mdespriee

select tablename, 
   HAS_TABLE_PRIVILEGE(tablename, 'select') as select,
   HAS_TABLE_PRIVILEGE(tablename, 'insert') as insert,
   HAS_TABLE_PRIVILEGE(tablename, 'update') as update,
   HAS_TABLE_PRIVILEGE(tablename, 'delete') as delete, 
   HAS_TABLE_PRIVILEGE(tablename, 'references') as references 
from pg_tables where schemaname='public' order by tablename;

mdespriee

export SBT_OPTS="-Xmx4G -XX:+UseConcMarkSweepGC -XX:+CMSClassUnloadingEnabled -XX:MaxPermSize=2G -Xss2M "

mdespriee

def normalize(s):
    """ Expects a unicode string, not encoded byte string.
        Returns unicode string
    """
    out = ''.join( c for c in unicodedata.normalize("NFKD", s)
                    if not unicodedata.combining(c) )
    out = _regexAlpha.sub(' ', out)
    out = _regexSpace.sub(' ', out)
    out = out.strip().upper()
    return out

mdespriee

cat /dev/urandom | tr -dc '[:alnum:]' | fold -w 64 | head -n 1

mdespriee

# grab the kernel module name
lshw -C network 2>&1 | grep wireless | grep driver

sudo modprobe -r ath9k && sudo modprobe ath9k

mdespriee

ssh edge_node "hdfs dfs -cat /some/path/part-*" | cat > file

cat file | ssh edge_node "hdfs dfs -put - /target/path"


# think of using a named pipe (mkfifo) to sream directly for application output 
rm -f stream
mkfifo stream

mdespriee

// This code is related to PR https://github.com/apache/spark/pull/17461
// I show how to use the setInitialModel() param of LDA to build a model incrementally, 
// and I compare the performance (perplexity) with a model built in one-shot


import scala.collection.mutable

import org.apache.spark.ml.{Pipeline, PipelineModel}
import org.apache.spark.ml.clustering.{LDA, LDAModel}

mdespriee

Project Title

One Paragraph of project description goes here

Getting Started

These instructions will get you a copy of the project up and running on your local machine for development and testing purposes. See deployment for notes on how to deploy the project on a live system.

Prerequisites

mdespriee

package ssp

import java.nio.charset.Charset
import java.nio.file.{Files, Paths}

import org.apache.spark.sql.functions.broadcast
import org.apache.spark.sql.streaming.{OutputMode, Trigger}
import org.apache.spark.sql.types.{StringType, StructField, StructType}
import org.apache.spark.sql.{Dataset, SparkSession}
import org.apache.spark.storage.StorageLevel

mdespriee

/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements.  See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License.  You may obtain a copy of the License at
*
*    http://www.apache.org/licenses/LICENSE-2.0
*