neo4j import test

ImportTest.java

import org.neo4j.graphdb.ResourceIterable;
import org.neo4j.graphdb.ResourceIterator;
import org.neo4j.io.fs.FileUtils;
import org.neo4j.kernel.impl.util.ResourceIterators;
import org.neo4j.kernel.logging.SystemOutLogging;
import org.neo4j.unsafe.impl.batchimport.Configuration;
import org.neo4j.unsafe.impl.batchimport.ParallelBatchImporter;
import org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory;
import org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdGenerator;
import org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdGenerators;
import org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMapper;
import org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMappers;
import org.neo4j.unsafe.impl.batchimport.input.Input;
import org.neo4j.unsafe.impl.batchimport.input.InputNode;
import org.neo4j.unsafe.impl.batchimport.input.InputRelationship;
import org.neo4j.unsafe.impl.batchimport.staging.CoarseUnboundedProgressExecutionMonitor;

import java.io.File;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * @author mh
 * @since 30.01.15
 */
// TODO fix Stackoverflow-Issue in Id-Mapping-Sorter
public class ImportTest {

    public static final String DIR = "target/test.db";
    private static final String[] LABELS = new String[] {"Person"};

    public static final int TOTAL_SIZE = 1_000_000_000;
    public static final int DOT_EVERY_N = 1_000_000;

    public static void main(String[] args) throws IOException {
        FileUtils.deleteRecursively(new File(DIR));
        CoarseUnboundedProgressExecutionMonitor monitor = new CoarseUnboundedProgressExecutionMonitor(DOT_EVERY_N, System.out);
        SystemOutLogging logging = new SystemOutLogging();
        ParallelBatchImporter importer = new ParallelBatchImporter(DIR, Configuration.DEFAULT, logging, monitor);
        // todo generate label and property id upfront
        importer.doImport(new OnlyNodeInput());
    }

    private static class OnlyNodeInput implements Input {

        @Override
        public ResourceIterable<InputNode> nodes() {

            return () -> new ResourceIterator<InputNode>() {
                AtomicInteger current = new AtomicInteger();

                @Override
                public void close() {
                }

                @Override
                public boolean hasNext() {
                    return current.get() < TOTAL_SIZE;
                }

                @Override
                public InputNode next() {
                    int id = current.getAndIncrement();
                    Object[] props = {"id", id};
                    return new InputNode(id, props,null,LABELS,null);
                }
            };
        }

        public ResourceIterable<InputRelationship> relationships() {
            return () -> ResourceIterators.EMPTY_ITERATOR;
        }

        public IdMapper idMapper() { return IdMappers.longs(NumberArrayFactory.OFF_HEAP); }

        public IdGenerator idGenerator() { return IdGenerators.startingFromTheBeginning(); }
    }
}

ImportTestRawId.java

import org.neo4j.graphdb.ResourceIterable;
import org.neo4j.graphdb.ResourceIterator;
import org.neo4j.io.fs.FileUtils;
import org.neo4j.kernel.impl.util.ResourceIterators;
import org.neo4j.kernel.logging.SystemOutLogging;
import org.neo4j.unsafe.impl.batchimport.Configuration;
import org.neo4j.unsafe.impl.batchimport.ParallelBatchImporter;
import org.neo4j.unsafe.impl.batchimport.cache.NumberArrayFactory;
import org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdGenerator;
import org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdGenerators;
import org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMapper;
import org.neo4j.unsafe.impl.batchimport.cache.idmapping.IdMappers;
import org.neo4j.unsafe.impl.batchimport.input.Input;
import org.neo4j.unsafe.impl.batchimport.input.InputNode;
import org.neo4j.unsafe.impl.batchimport.input.InputRelationship;
import org.neo4j.unsafe.impl.batchimport.staging.CoarseUnboundedProgressExecutionMonitor;

import java.io.File;
import java.io.IOException;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * @author mh
 * @since 30.01.15
 */
public class ImportTestRawId {

    public static final String DIR = "target/test.db";
    private static final String[] LABELS = new String[] {"Person"};

    public static final int TOTAL_SIZE = 1_000_000_000;
    public static final int DOT_EVERY_N = 1_000_000;

    public static void main(String[] args) throws IOException {
        FileUtils.deleteRecursively(new File(DIR));
        CoarseUnboundedProgressExecutionMonitor monitor = new CoarseUnboundedProgressExecutionMonitor(DOT_EVERY_N, System.out);
        SystemOutLogging logging = new SystemOutLogging();
        ParallelBatchImporter importer = new ParallelBatchImporter(DIR, Configuration.DEFAULT, logging, monitor);
        // todo generate label and property id upfront
        importer.doImport(new OnlyNodeInput());
    }

    private static class OnlyNodeInput implements Input {

        @Override
        public ResourceIterable<InputNode> nodes() {

            return () -> new ResourceIterator<InputNode>() {
                AtomicInteger current = new AtomicInteger();

                @Override
                public void close() {
                }

                @Override
                public boolean hasNext() {
                    return current.get() < TOTAL_SIZE;
                }

                @Override
                public InputNode next() {
                    long id = current.getAndIncrement();
                    Object[] props = {"id", id};
                    return new InputNode(id, props,null,LABELS,null);
                }
            };
        }

        public ResourceIterable<InputRelationship> relationships() {
            return () -> ResourceIterators.EMPTY_ITERATOR;
        }

        public IdMapper idMapper() {
            return IdMappers.actual();
        }

        public IdGenerator idGenerator() {
            return IdGenerators.fromInput();
        }
    }
}

pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    <modelVersion>4.0.0</modelVersion>

    <groupId>org.neo4j</groupId>
    <artifactId>neo4j-import-test</artifactId>
    <version>1.0-SNAPSHOT</version>

    <dependencies>
        <dependency>
            <groupId>org.neo4j</groupId>
            <artifactId>neo4j-kernel</artifactId>
            <version>2.2.0-M03</version>
        </dependency>
    </dependencies>

</project>

run.sh

MAVEN_OPTS="-Xmx8G" mvn clean compile -Dexec.mainClass=ImportTestRawId