Presto UDFs integration test

An integration test for Presto (UDF) plugin

Reference files for integration test of a Presto plugin containing hashing-related UDFs

Dockerfile

FROM openjdk:8
LABEL maintainer="JumboTech <[email protected]>"
LABEL Description="Presto Fluentd docker image" Vendor="company Organization" Version="0.1"

ARG jarFile

ENV HOME /opt/presto
ENV PRESTO_VERSION 0.194
ENV JAVA_HOME /usr/java/default
ENV PATH $PATH:$JAVA_HOME/bin

RUN apt-get update && apt-get install -y \
    curl \
    vim \
    python2.7 \
    bash \
    htop \
    sudo \
    && rm -rf /var/lib/apt/lists/*

RUN mkdir -p /opt/presto/plugin/company-plugin && \
    curl https://repo1.maven.org/maven2/com/facebook/presto/presto-server/${PRESTO_VERSION}/presto-server-${PRESTO_VERSION}.tar.gz -o presto-server.tar.gz && \
    tar xfz  presto-server.tar.gz -C /opt/presto --strip-components=1 && \
    rm presto-server.tar.gz

ADD https://repo1.maven.org/maven2/com/facebook/presto/presto-cli/${PRESTO_VERSION}/presto-cli-${PRESTO_VERSION}-executable.jar /usr/local/bin/presto-cli
RUN chmod a+x /usr/local/bin/presto-cli

RUN curl -L https://toolbelt.treasuredata.com/sh/install-ubuntu-xenial-td-agent3.sh | sh

COPY nonsvn/presto/etc/ /opt/presto/etc/
COPY nonsvn/td-agent/* /etc/td-agent/
COPY nonsvn/docker-entrypoint.sh /opt/presto/

ADD target/${jarFile} /opt/presto/plugin/company-plugin/${jarFile}

EXPOSE 8889

ENTRYPOINT ["/opt/presto/docker-entrypoint.sh"]

ExtendedHashFunctionsTestIT.java

package com.company.plugin.it;

import com.spotify.docker.client.DefaultDockerClient;
import com.spotify.docker.client.DockerClient;
import com.spotify.docker.client.LogStream;
import com.spotify.docker.client.messages.*;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;

import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import static com.company.plugin.it.Config.*;
import static org.junit.Assert.fail;

public class ExtendedHashFunctionsTestIT {

    public DockerClient docker;
    public String id;

    @BeforeClass
    public void setupDocker() {
        try {
            docker = DefaultDockerClient.fromEnv().build();

            final ImageInfo docker_image = docker.inspectImage(IMAGE_NAME);

            final String[] ports = {HOST_PORT, CONTAINER_PORT};
            final Map<String, List<PortBinding>> portBindings = new HashMap<>();
            for (String port : ports) {
                List<PortBinding> hostPorts = new ArrayList<>();
                hostPorts.add(PortBinding.of(LOCALHOST, port));
                portBindings.put(port, hostPorts);
            }

            final HostConfig hostConfig = HostConfig.builder()
                    .portBindings(portBindings)
                    .build();

            final ContainerConfig containerConfig = ContainerConfig.builder()
                    .hostConfig(hostConfig)
                    .image(docker_image.id())
                    .exposedPorts(ports)
                    .cmd("sh", "-c", "while :; do sleep 1; done")
                    .build();

            final ContainerCreation creation = docker.createContainer(containerConfig);
            id = creation.id();

            final ContainerInfo info = docker.inspectContainer(id);

            docker.startContainer(id);

            final String[] command = {"sh", "-c", "ls"};
            final ExecCreation execCreation = docker.execCreate(
                    id, command, DockerClient.ExecCreateParam.attachStdout(),
                    DockerClient.ExecCreateParam.attachStderr());
            final LogStream output = docker.execStart(execCreation.id());

            Thread.sleep(PRESTO_SLEEP_TIME);
        } catch (Exception exception) {
            exception.printStackTrace();
        }
    }

    @Test
    public void test() {
        try (Connection connection = DriverManager.getConnection(PRESTO_URL, PRESTO_USER, PRESTO_PASSWORD);
             Statement statement = connection.createStatement()) {
            ResultSet resultSet = statement.executeQuery("SELECT SHA1(9223372036854775807) AS hash");
            assertEquals("458B642B137E2C76E0B746C6FA43E64C3D4C47F1", resultSet.getString("hash"));
        } catch (Exception ex) {
            ex.printStackTrace();
            fail("Exception occured");
        }
    }

    @AfterClass
    public void destroyDocker() {
        try {
            docker.killContainer(id);
            docker.removeContainer(id);
            docker.close();

        } catch (Exception exception) {
            exception.printStackTrace();
        }
    }
}

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>com.company.plugin</groupId>
    <artifactId>company-plugin</artifactId>
    <version>1.1.0</version>
    <packaging>jar</packaging>
    <name>company-Presto-Plugin</name>

    <properties>
        <presto.version>0.194</presto.version>
        <tempto.version>1.46</tempto.version>
        <jackson.version>2.9.4</jackson.version>
        <jersey.version>2.26</jersey.version>
    </properties>

    <dependencies>
        <dependency>
            <groupId>com.facebook.presto</groupId>
            <artifactId>presto-spi</artifactId>
            <version>${presto.version}</version>
        </dependency>

        <dependency>
            <groupId>io.prestodb.tempto</groupId>
            <artifactId>tempto-core</artifactId>
            <version>${tempto.version}</version>
            <exclusions>
                <exclusion>
                    <groupId>com.datastax.cassandra</groupId>
                    <artifactId>cassandra-driver-core</artifactId>
                </exclusion>
                <exclusion>
                    <groupId>com.google.code.findbugs</groupId>
                    <artifactId>annotations</artifactId>
                </exclusion>
            </exclusions>
        </dependency>

        <dependency>
            <groupId>io.prestodb.tempto</groupId>
            <artifactId>tempto-ldap</artifactId>
            <version>${tempto.version}</version>
        </dependency>

        <dependency>
            <groupId>io.prestodb.tempto</groupId>
            <artifactId>tempto-runner</artifactId>
            <version>${tempto.version}</version>
        </dependency>

        <dependency>
            <groupId>com.facebook.presto</groupId>
            <artifactId>presto-jdbc</artifactId>
            <version>${presto.version}</version>
        </dependency>

        <dependency>
            <groupId>io.airlift</groupId>
            <artifactId>log</artifactId>
            <version>0.148</version>
        </dependency>

        <dependency>
            <groupId>com.google.guava</groupId>
            <artifactId>guava</artifactId>
            <version>23.0</version>
        </dependency>

        <dependency>
            <groupId>org.fluentd</groupId>
            <artifactId>fluent-logger</artifactId>
            <version>0.3.1</version>
        </dependency>

        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-databind</artifactId>
            <version>${jackson.version}</version>
        </dependency>

        <dependency>
            <groupId>com.fasterxml.jackson.core</groupId>
            <artifactId>jackson-annotations</artifactId>
            <version>${jackson.version}</version>
        </dependency>

        <!-- udfs -->
        <dependency>
            <groupId>com.facebook.presto</groupId>
            <artifactId>presto-main</artifactId>
            <version>${presto.version}</version>
            <scope>provided</scope>
        </dependency>

        <dependency>
            <groupId>io.airlift</groupId>
            <artifactId>slice</artifactId>
            <version>0.36</version>
            <scope>provided</scope>
        </dependency>

        <dependency>
            <groupId>io.airlift</groupId>
            <artifactId>testing</artifactId>
            <version>0.172</version>
            <scope>provided</scope>
        </dependency>
        <!-- udfs -->

        <!-- version conflict resolution -->
        <dependency>
            <groupId>org.glassfish.jersey.core</groupId>
            <artifactId>jersey-client</artifactId>
            <version>${jersey.version}</version>
            <scope>provided</scope>
        </dependency>

        <dependency>
            <groupId>org.glassfish.jersey.connectors</groupId>
            <artifactId>jersey-apache-connector</artifactId>
            <version>${jersey.version}</version>
            <scope>provided</scope>
        </dependency>

        <dependency>
            <groupId>org.glassfish.jersey.media</groupId>
            <artifactId>jersey-media-json-jackson</artifactId>
            <version>${jersey.version}</version>
            <scope>provided</scope>
        </dependency>

        <dependency>
            <groupId>org.glassfish.jersey.ext</groupId>
            <artifactId>jersey-entity-filtering</artifactId>
            <version>${jersey.version}</version>
            <scope>provided</scope>
        </dependency>
        <!-- version conflict resolution -->

        <!-- for testing -->
        <dependency>
            <groupId>junit</groupId>
            <artifactId>junit</artifactId>
            <version>4.11</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>org.junit.jupiter</groupId>
            <artifactId>junit-jupiter-api</artifactId>
            <version>5.0.3</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>org.testng</groupId>
            <artifactId>testng</artifactId>
            <version>6.9.8</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>com.spotify</groupId>
            <artifactId>docker-client</artifactId>
            <version>8.11.1</version>
            <scope>test</scope>
        </dependency>

        <dependency>
            <groupId>org.slf4j</groupId>
            <artifactId>slf4j-simple</artifactId>
            <version>1.7.22</version>
            <scope>test</scope>
        </dependency>

        <!-- udfs -->
        <dependency>
            <groupId>com.facebook.presto</groupId>
            <artifactId>presto-main</artifactId>
            <version>${presto.version}</version>
            <type>test-jar</type>
            <scope>test</scope>
        </dependency>
        <!-- udfs -->
        <!-- for testing -->

        <!-- dependency analyis -->
        <dependency>
            <groupId>org.apache.maven.plugins</groupId>
            <artifactId>maven-dependency-plugin</artifactId>
            <version>3.1.1</version>
        </dependency>
        <!-- dependency analyis -->
    </dependencies>

    <build>
        <plugins>
            <plugin>
                <groupId>org.codehaus.mojo</groupId>
                <artifactId>build-helper-maven-plugin</artifactId>
                <version>3.0.0</version>
                <executions>
                    <execution>
                        <id>add-test-source</id>
                        <phase>process-resources</phase>
                        <goals>
                            <goal>add-test-source</goal>
                        </goals>
                        <configuration>
                            <sources>
                                <source>src/it/java</source>
                            </sources>
                        </configuration>
                    </execution>
                </executions>
            </plugin>

            <plugin>
                <artifactId>maven-compiler-plugin</artifactId>
                <version>3.6.1</version>
                <configuration>
                    <source>1.8</source>
                    <target>1.8</target>
                    <encoding>UTF-8</encoding>
                </configuration>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-assembly-plugin</artifactId>
                <version>3.1.0</version>
                <configuration>
                    <descriptorRefs>
                        <descriptorRef>jar-with-dependencies</descriptorRef>
                    </descriptorRefs>
                </configuration>
                <executions>
                    <execution>
                        <id>assemble-all</id>
                        <phase>package</phase>
                        <goals>
                            <goal>single</goal>
                        </goals>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>org.apache.maven.plugins</groupId>
                <artifactId>maven-failsafe-plugin</artifactId>
                <version>2.5</version>
                <executions>
                    <execution>
                        <goals>
                            <goal>integration-test</goal>
                            <goal>verify</goal>
                        </goals>
                        <configuration>
                            <includes>
                                <include>**/*.java</include>
                            </includes>
                        </configuration>
                    </execution>
                </executions>
            </plugin>
            <plugin>
                <groupId>com.spotify</groupId>
                <artifactId>dockerfile-maven-plugin</artifactId>
                <version>1.3.7</version>
                <executions>
                    <execution>
                        <id>default</id>
                        <goals>
                            <goal>build</goal>
                            <goal>push</goal>
                        </goals>
                    </execution>
                    <execution>
                        <id>start</id>
                        <phase>pre-integration-test</phase>
                    </execution>
                </executions>
                <configuration>
                    <repository>jumbo/prestodb</repository>
                    <tag>latest</tag>
                    <imageTags>
                        <imageTag>${project.version}</imageTag>
                    </imageTags>
                    <buildArgs>
                        <jarFile>company-plugin-${project.version}-jar-with-dependencies.jar</jarFile>
                    </buildArgs>
                </configuration>
            </plugin>
        </plugins>
    </build>

</project>