nddipiazza

<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:soap="http://schemas.microsoft.com/sharepoint/soap/">
   <soapenv:Header/>
   <soapenv:Body>
      <soap:GetContent>
         <soap:objectType>Site</soap:objectType>
         <soap:retrieveChildItems>true</soap:retrieveChildItems>
         <soap:securityOnly>true</soap:securityOnly>
      </soap:GetContent>
   </soapenv:Body>
</soapenv:Envelope>

nddipiazza

1519885689195	geckodriver	INFO	geckodriver 0.19.1
151988155618998189576	8g9e1c9k5o	dgreicvkeord	rIiNvFeOr		LIiNsFtOe	ngiencgk oodnr i12v7e.r0 .00..119:.219
1519885865809
197	geckodriver	INFO	Listening on 127.0.0.1:11831
*** You are running in headless mode.
*** You are running in headless mode.
Mar 01, 2018 6:28:10 AM org.openqa.selenium.remote.ProtocolHandshake createSession
INFO: Detected dialect: W3C
Mar 01, 2018 6:28:10 AM org.openqa.selenium.remote.ProtocolHandshake createSession
INFO: Detected dialect: W3C

nddipiazza

/home/ndipiazza/Downloads/jdk1.8.0_141/bin/java -agentlib:jdwp=transport=dt_socket,address=127.0.0.1:45155,suspend=y,server=n -javaagent:/home/ndipiazza/Downloads/idea-IC-173.4127.27/plugins/Groovy/lib/agent/gragent.jar -javaagent:/home/ndipiazza/Downloads/idea-IC-173.4127.27/lib/rt/debugger-agent.jar=/tmp/capture.props -Dfile.encoding=UTF-8 -classpath /home/ndipiazza/Downloads/jdk1.8.0_141/jre/lib/charsets.jar:/home/ndipiazza/Downloads/jdk1.8.0_141/jre/lib/deploy.jar:/home/ndipiazza/Downloads/jdk1.8.0_141/jre/lib/ext/cldrdata.jar:/home/ndipiazza/Downloads/jdk1.8.0_141/jre/lib/ext/dnsns.jar:/home/ndipiazza/Downloads/jdk1.8.0_141/jre/lib/ext/jaccess.jar:/home/ndipiazza/Downloads/jdk1.8.0_141/jre/lib/ext/jfxrt.jar:/home/ndipiazza/Downloads/jdk1.8.0_141/jre/lib/ext/localedata.jar:/home/ndipiazza/Downloads/jdk1.8.0_141/jre/lib/ext/nashorn.jar:/home/ndipiazza/Downloads/jdk1.8.0_141/jre/lib/ext/sunec.jar:/home/ndipiazza/Downloads/jdk1.8.0_141/jre/lib/ext/sunjce_provider.jar:/home/ndipiazza/Downloads/jdk1.8.0_141/jr

nddipiazza

sudo docker run -t -d -p 8765:8765 -p 8984:8984 -p 8983:8983 -p 8764:8764 --shm-size=2g --name testzombies openjdk:8u151-jdk
sudo docker exec -i -t testzombies /bin/bash
apt-get -y --fix-missing update && \
  apt-get -y --fix-missing upgrade && \
  apt-get -y --no-install-recommends install apt-utils && \
  apt-get install -y wget lbzip2 git openjdk-8-jdk libdbus-glib-1-2 xvfb libgtk-3.0 procps gradle psmisc
wget https://download-installer.cdn.mozilla.net/pub/devedition/releases/59.0b14/linux-x86_64/en-US/firefox-59.0b14.tar.bz2
tar xvf firefox-59.0b14.tar.bz2
wget https://github.com/mozilla/geckodriver/releases/download/v0.19.1/geckodriver-v0.19.1-linux64.tar.gz
tar xzvf geckodriver-v0.19.1-linux64.tar.gz 

nddipiazza

diff --git a/testing/marionette/cookie.js b/testing/marionette/cookie.js
--- a/testing/marionette/cookie.js
+++ b/testing/marionette/cookie.js
@@ -144,14 +144,14 @@ cookie.add = function(newCookie, {restri
     newCookie.domain = "." + newCookie.domain;
   }
 
-  if (restrictToHost) {
-    if (!restrictToHost.endsWith(newCookie.domain) &&
-        ("." + restrictToHost) !== newCookie.domain &&

nddipiazza

<#
    .Synopsis 
        Retrieve SPOIDCR cookie for SharePoint Online.
    .Description
        Authenticates to the sts and retrieves the SPOIDCR cookie for SharePoint Online.
        Will use the custom IDP if one has been setup.
        Optionally, can use integrated credentials (when integrated is set to true) with ADFS using the windowsmixed endpoint.
        Results are formattable as XML, JSON, KEYVALUE, and by line.
        
        Makes global variables avaiable at the end of the run.

nddipiazza

Here is a picture of sharepoint web architecture: https://pasteboard.co/HkyuMhZ.png

We need to crawl groups at each of the site collections. Each can have unique sharepoint groups.

Example of site collections in a web application

http://example.sharepoint.com/

nddipiazza

import crawler.common.sharepoint.stubs.sitedata.ArrayOfSList;
import crawler.common.sharepoint.stubs.sitedata.GetListCollectionResponse;
import crawler.common.sharepoint.stubs.sitedata.SList;
import crawler.common.sharepoint.stubs.sitedata.SiteData;
import crawler.common.sharepoint.stubs.sitedata.SiteDataSoap;
import org.apache.cxf.configuration.security.AuthorizationPolicy;
import org.apache.cxf.endpoint.Client;
import org.apache.cxf.frontend.ClientProxy;
import org.apache.cxf.transport.http.HTTPConduit;
import org.apache.cxf.transport.http.asyncclient.AsyncHTTPConduit;

nddipiazza

-Dcom.lucidworks.fusion.plugin.portStart=8871 -Dcom.lucidworks.fusion.plugin.portEnd=8971 -DserviceName=connectors-rpc -Dcom.lucidworks.apollo.solr.zk.connect=localhost:9983/lwfusion/4.2.0-SNAPSHOT/solr -Dcurator.zk.connect=localhost:9983 -Dcurator.namespace=lwfusion/4.2.0-SNAPSHOT/core -Dcom.lucidworks.fusion.plugin.portStart=8871 -Dcom.lucidworks.fusion.plugin.portEnd=8971 -DserviceName=connectors-rpc -Dcom.lucidworks.apollo.solr.zk.connect=localhost:9983/lwfusion/4.2.0-SNAPSHOT/solr -Dcurator.zk.connect=localhost:9983 -Dcurator.namespace=lwfusion/4.2.0-SNAPSHOT/core

nddipiazza

import com.google.common.collect.Sets;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.LineIterator;

import java.io.File;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;

public class DiffTextFilesUtil {