| import json | |
| import urllib2 as url | |
| import pandas as pnd | |
| import numpy as np | |
| # Get the data | |
| json_url="http://knoema.com/api/1.0/data/FAOFSDFI2012?Time=2000-2007&food-item=1000030,1000520,1000120,1000080,1000110,1000540,1000550&country=1000550,1000560,1000770,1001680,1001660,1000010,1000020,1000030,1000040,1000050,1000060,1000070,1000080,1000090,1000100,1000110,1000120,1000130,1000140,1000150,1000160,1000170,1000180,1000190,1000200,1000210,1000220,1000230,1000240,1000250,1000260,1000270,1000280,1000290,1000300" \ | |
| ",1000310,1000320,1000330,1000340,1000350,1000360,1000370,1000380,1000390,1000400,1000410,1000420,1000430,1000440,1000450,1000460,1000470,1000480,1000490,1000500,1000510,1000520,1000530,1000540,1000570,1000580,1000590,1000600,1000610,1000620,1000630,1000640,1000650,1000660,1000670,1000680,1000690,1000700,1000710,1000720,1000730,1000740,1000750,1000760,1000780,1000790,1000800,1000810,1000820,1000830,1000840,1000850" \ | |
| ",1000860,1000870,1000880,1000890,1000900,1000910,1000920,1000930,100 |
| // Gets the schema from the CSV files | |
| val crimesSchema = spark.read | |
| .option("inferSchema", true) | |
| .option("header","true") | |
| .csv(dataPath) | |
| .schema | |
| // Creates Structured Stream over the CSV files | |
| val crimes = spark.readStream | |
| .schema(crimesSchema) |
| Exception in thread "main" java.nio.file.NoSuchFileException: /usr/share/elasticsearch/plugins/.installing-2175999155815230137/plugin-descriptor.properties | |
| at sun.nio.fs.UnixException.translateToIOException(UnixException.java:86) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:102) | |
| at sun.nio.fs.UnixException.rethrowAsIOException(UnixException.java:107) | |
| at sun.nio.fs.UnixFileSystemProvider.newByteChannel(UnixFileSystemProvider.java:214) |
| sg_index_1: | |
| indices: | |
| 'index_1': | |
| '*': | |
| - READ | |
| - SEARCH | |
| - 'indices:data/read/search' | |
| sg_index_2: | |
| indices: | |
| 'index_2': |
| import pandas as pd | |
| # Search '>' over all the values of the whole dataframe | |
| res = df.apply(lambda row: row.astype(str).str.contains('>').any(), axis=1) |
| docker run --rm --link elastic:elasticsearch -p 5601:5601 -e ELASTICSEARCH_USERNAME=elastic -e ELASTICSEARCH_PASSWORD=elastic docker.elastic.co/kibana/kibana:7.6.0 | |
| docker run --rm -p 9200:9200 -p 9300:9300 --name elastic -e "xpack.security.enabled=true" -v /home/netanel/elastic/secrets:/run/secrets -e ELASTIC_PASSWORD_FILE=/run/secrets/bootstrapPassword.txt -e "discovery.type=single-node" docker.elastic.co/elasticsearch/elasticsearch:7.6.0 |
| Uploading to apache.snapshots.https: https://repository.apache.org/content/repositories/snapshots/org/apache/sedona/sedona-parent/1.0.0-incubator-SNAPSHOT/sedona-parent-1.0.0-incubator-20201221.094716-10.pom | |
| [INFO] ------------------------------------------------------------------------ | |
| [INFO] Reactor Summary for sedona-parent 1.0.0-incubator-SNAPSHOT: | |
| [INFO] | |
| [INFO] sedona-parent ...................................... FAILURE [ 8.376 s] | |
| [INFO] org.apache.sedona:sedona-core_2.12 ................. SKIPPED | |
| [INFO] org.apache.sedona:sedona-sql-3.0_2.12 .............. SKIPPED | |
| [INFO] org.apache.sedona:sedona-viz-3.0_2.12 .............. SKIPPED | |
| [INFO] org.apache.sedona:sedona-python-adapter-3.0_2.12 ... SKIPPED | |
| [INFO] ------------------------------------------------------------------------ |
| ==== test size of class org.locationtech.jts.index.quadtree.Quadtree==== | |
| WKB: SHAPE: | |
| original size : 1726447 1723815 | |
| with serde kryo size : 588667 497407 | |
| percent : 0.34097021223356405 0.2885501054347479 | |
| ==== test size of class org.locationtech.jts.index.strtree.STRtree==== | |
| WKB: SHAPE: | |
| original size : 1521545 1723815 | |
| with serde kryo size : 610013 497407 |
