Barney's NY Web Crawler Script

barneys.py

#!/usr/bin/env python

import datetime
import inspect
import json
import lxml.html
from lxml.cssselect import CSSSelector
import re
import requests
import os
import sys
import time
import trollius
import tweepy

from random import randint
from destroyer import agent

requests.packages.urllib3.disable_warnings()
current_milli_time = lambda: int(round(time.time() * 1000))

sem=trollius.Semaphore(50)

########################################################################
productId="504548893" #Pharrell NMDs
########################################################################

variants=[]

array_email=[]
array_first_name=[]
array_last_name=[]
array_address1=[]
array_address2=[]
array_city=[]
array_state_code=[]
array_postal_code=[]
array_phone=[]
array_country_code=[]

array_ccType=[] #MasterCard #Visa #Amex #Discover #BN
array_ccM=[]
array_ccYYYY=[]
array_ccNumber=[]
array_ccCCV=[]
array_ccName=[]

array_sizeList=[]
array_proxy=[]

########################################################################
array_email.append('[email protected]')
array_first_name.append('First')
array_last_name.append('Last')
array_address1.append('Address1')
array_address2.append('Address2')
array_city.append('New York')
array_state_code.append('NY')
array_postal_code.append('10012')
array_phone.append('212-867-5309')
array_country_code.append('US')
array_ccType.append('Visa')
array_ccM.append('5')
array_ccYYYY.append('2017')
array_ccNumber.append('4111 111 111 111')
array_ccCCV.append('123')
array_ccName.append('Johnny A Cardholder')
array_sizeList.append([['9','8.5']])
array_proxy.append(None)
########################################################################

#Critcal - need IPs that work.
proxyListCycle=[
  "username:password@ip:port",
]

########################################################################

def lineno():
  """Returns the current line number in our program."""
  return inspect.currentframe().f_back.f_lineno

def supreme(destroyerID):
  proxies=array_proxy[destroyerID]

  submitJSON=None

  h_="Destroyer # "+str(destroyerID).rjust(4," ")+" >> "

  start_time_ = time.time()
  barneySession=requests.Session()
  barneySession.mount("http://", requests.adapters.HTTPAdapter(max_retries=1))
  barneySession.mount("https://", requests.adapters.HTTPAdapter(max_retries=1))
  barneySession.verify=False
  barneySession.proxies=proxies

  barneySession.cookies.clear()

  email=array_email[destroyerID]
  first_name=array_first_name[destroyerID]
  last_name=array_last_name[destroyerID]
  address1=array_address1[destroyerID]
  address2=array_address2[destroyerID]
  city=array_city[destroyerID]
  state_code=array_state_code[destroyerID]
  postal_code=array_postal_code[destroyerID]
  phone=array_phone[destroyerID]
  country_code=array_country_code[destroyerID]

  ccType=array_ccType[destroyerID] #MasterCard #Visa #Amex #Discover #BN
  ccM=array_ccM[destroyerID]
  ccYYYY=array_ccYYYY[destroyerID]
  ccNumber=array_ccNumber[destroyerID]
  ccCCV=array_ccCCV[destroyerID]
  ccName=array_ccName[destroyerID]

  itemNumber=0

  sizeList=array_sizeList[destroyerID][itemNumber]

  full_name=first_name+" "+last_name

  mySku=None
  myShippingKey=None

  try:
    for size in sizeList:
      for variant in variants:
        if size.lower() == variant[0].lower():
          if variant[2] < 1:
            continue

          url="https://www.barneys.com/on/demandware.store/Sites-BNY-Site/default/API-GetSerenadeInventory?ProductID="+str(variant[1])
          headers={
            'User-Agent':'Barneys/26 (iPhone; iOS 9.1; Scale/3.00)',
            'x-dw-client-id':'860f00af-81a2-4e4f-9dda-292b25d3ed94',
            'Cookie':'ak_bmsc=',
          }

          response=barneySession.get(url,headers=headers)
          inventoryJSON=json.loads(response.text)
          print str(h_+"Sz - " + str(size)).ljust(12," ") + " : " + "Available" + " : " + str(inventoryJSON['feedback']['availableQty'])

          if inventoryJSON['feedback']['availableQty'] > 0:
            mySku=str(variant[1])
            raise StopIteration
  except StopIteration:
    pass
  except:
    mySku=None
    pass


  if mySku is not None:
    try:
      headers={
        'User-Agent':'Barneys/12 CFNetwork/758.4.3 Darwin/15.5.0',
        'x-dw-client-id':'860f00af-81a2-4e4f-9dda-292b25d3ed94',
        'Content-Type':'application/json; charset=utf-8',
        'Cookie':'ak_bmsc=',
        'Connection': 'keep-alive',
      }
      url="https://www.barneys.com/s/BNY/dw/shop/v13_6/basket/this/add"
      data={
        "product_id": mySku,
        "quantity": 1
      }
      response=barneySession.post(url,data=json.dumps(data),headers=headers)

     #print response.text

      cartJSON=json.loads(response.text)
      print h_+"Carted".ljust(12," ") + " : " + str(cartJSON['product_items'][-1]['item_text'])
      print h_+"Product ID".ljust(12," ") + " : " + str(cartJSON['product_items'][-1]['product_id'])
      print h_+"Quantity".ljust(12," ") + " : " + str(cartJSON['product_items'][-1]['quantity'])
      print h_+"Order Total".ljust(12," ") + " : " + str(cartJSON['order_total']) + " " + str(cartJSON['currency'])

      url="https://www.barneys.com/s/BNY/dw/shop/v13_6/basket/this/checkout"
      headers={
        'User-Agent':'Barneys/12 CFNetwork/758.4.3 Darwin/15.5.0',
        'x-dw-client-id':'860f00af-81a2-4e4f-9dda-292b25d3ed94',
        'Content-Type':'application/json; charset=utf-8',
        'Connection': 'keep-alive',
      }
      response=barneySession.get(url,headers=headers)
      etag=response.headers['ETag']
      checkoutJSON=json.loads(response.text)
      """
      for _flash in checkoutJSON['_flash']:
        print _flash['type']
      print
      """

      url="https://www.barneys.com/s/BNY/dw/shop/v13_6/basket/this/checkout/set_shipping_address"
      headers={
        'User-Agent':'Barneys/26 (iPhone; iOS 9.1; Scale/3.00)',
        'x-dw-client-id':'860f00af-81a2-4e4f-9dda-292b25d3ed94',
        'Content-Type':'application/json; charset=utf-8',
        'If-Match': etag,
        'Connection': 'keep-alive',
      }
      data={
        "address1": address1,
        "address2": address2,
        "city": city,
        "country_code": country_code,
        "first_name": first_name,
        "full_name": full_name,
        "last_name": last_name,
        "phone": phone,
        "postal_code": postal_code,
        "state_code": state_code
      }
      response=barneySession.post(url,data=json.dumps(data),headers=headers)

      url="https://www.barneys.com/s/BNY/dw/shop/v13_6/basket/this/checkout"
      response=barneySession.get(url,headers=headers)
      etag=response.headers['ETag']
      checkoutJSON=json.loads(response.text)
      """
      for _flash in checkoutJSON['_flash']:
        print _flash['type']
      print
      """

      url="https://www.barneys.com/s/BNY/dw/shop/v13_6/basket/this/checkout/set_customer_info"
      headers={
        'User-Agent':'Barneys/26 (iPhone; iOS 9.1; Scale/3.00)',
        'x-dw-client-id':'860f00af-81a2-4e4f-9dda-292b25d3ed94',
        'Content-Type':'application/json; charset=utf-8',
        'If-Match': etag,
        'Connection': 'keep-alive',
      }
      data={
        "email": email,
      }
      response=barneySession.post(url,data=json.dumps(data),headers=headers)
      etag=response.headers['ETag']

      url="https://www.barneys.com/s/BNY/dw/shop/v13_6/basket/this/checkout"
      response=barneySession.get(url,headers=headers,proxies=proxies)
      etag=response.headers['ETag']
      checkoutJSON=json.loads(response.text)
      """
      for _flash in checkoutJSON['_flash']:
        print _flash['type']
      print
      """

      url="https://www.barneys.com/s/BNY/dw/shop/v13_6/basket/this/checkout/set_billing_address"
      headers={
        'User-Agent':'Barneys/26 (iPhone; iOS 9.1; Scale/3.00)',
        'x-dw-client-id':'860f00af-81a2-4e4f-9dda-292b25d3ed94',
        'Content-Type':'application/json; charset=utf-8',
        'If-Match': etag,
        'Connection': 'keep-alive',
      }
      data={
        "address1": address1,
        "address2": address2,
        "city": city,
        "country_code": country_code,
        "first_name": first_name,
        "last_name": last_name,
        "phone": phone,
        "postal_code": postal_code,
        "state_code": state_code
      }
      response=barneySession.post(url,data=json.dumps(data),headers=headers)

      url="https://www.barneys.com/s/BNY/dw/shop/v13_6/basket/this/checkout"
      response=barneySession.get(url,headers=headers)
      etag=response.headers['ETag']
      checkoutJSON=json.loads(response.text)
      """
      for _flash in checkoutJSON['_flash']:
        print _flash['type']
      print
      """

      url="https://www.barneys.com/on/demandware.store/Sites-BNY-Site/default/API-AddCreditCardToBasket"
      headers={
        'User-Agent':'Barneys/26 (iPhone; iOS 9.1; Scale/3.00)',
      }

      if ccType == "BN":

        data={
          "number": ccNumber.replace(" ",""),
          "owner": ccName,
          "type": ccType,
        }

      else:

        data={
          "cvn": ccCCV,
          "month": ccM,
          "number": ccNumber.replace(" ",""),
          "owner": ccName,
          "type": ccType,
          "year": ccYYYY,
        }

      response=barneySession.post(url,data=data,headers=headers)
      ccJSON=json.loads(response.text)
      print h_+"Status".ljust(12," ") + " : " + ccJSON['status']

      if "invalid" in ccJSON['status']:
        return destroyerID, size, first_name, last_name, email, ccNumber, submitJSON, str(ccJSON['status'])

      url="https://www.barneys.com/s/BNY/dw/shop/v13_6/basket/this/checkout"
      headers={
        'User-Agent':'Barneys/26 (iPhone; iOS 9.1; Scale/3.00)',
        'x-dw-client-id':'860f00af-81a2-4e4f-9dda-292b25d3ed94',
        'Connection': 'keep-alive',
      }
      response=barneySession.get(url,headers=headers)
      etag=response.headers['ETag']
      checkoutJSON=json.loads(response.text)

      """
      print json.dumps(checkoutJSON,indent=2)
      """

      url="https://www.barneys.com/on/demandware.store/Sites-BNY-Site/default/API-SubmitBasket"
      headers={
        'User-Agent':'Barneys/26 (iPhone; iOS 9.1; Scale/3.00)',
        'Authorization':'Basic c3RvcmVmcm9udDpiYXJuZXlz',
        'Connection': 'keep-alive',
      }
      response=barneySession.get(url,headers=headers)
      submitJSON=json.loads(response.text)
      print h_+str(submitJSON)

      return destroyerID, size, first_name, last_name, email, ccNumber, submitJSON
    except:
      pass

  else:
    return destroyerID, size, first_name, last_name, email, ccNumber, submitJSON, "SKU not found"

  return destroyerID, size, first_name, last_name, email, ccNumber, submitJSON, "Exception"

def destroy():
  loop = trollius.get_event_loop()
  futures = []
  for destroyerID in range(0,len(array_first_name)):
    futures.append(loop.run_in_executor(None, supreme, destroyerID))
  return futures

@trollius.coroutine
def call():
  results = []
  futures = destroy()
  f=open(str(current_milli_time())+".atc",'w')
  f.write(sys.argv[0]+"\n")
  f.write(str(datetime.datetime.now())+"\n")
  for future in futures:
      result = yield trollius.From(future)

      print
      print result
      f.write(str(result)+"\n")
      print

      results.append(result)
  f.close()
  raise trollius.Return(results)

#**********************************************************************#
#  Main
#**********************************************************************#

if __name__ == "__main__":

  ########################################################################
  # Loop through store inventory until it changes.
  ########################################################################

  cycleJSON=True

  iteration=0

  while cycleJSON:

    session=requests.Session()
    session.mount("http://", requests.adapters.HTTPAdapter(max_retries=1))
    session.mount("https://", requests.adapters.HTTPAdapter(max_retries=1))
    session.verify=False
    session.cookies.clear()
    proxies={
      "http" : "http://"+proxyListCycle[iteration%len(proxyListCycle)],
      "https": "https://"+proxyListCycle[iteration%len(proxyListCycle)]
    }

    start_time = time.time()

    url="http://www.barneys.com/on/demandware.store/Sites-BNY-Site/default/Product-GetVariants?pid="+str(productId)

    headers={
      'User-Agent':'Barneys/26 (iPhone; iOS 9.1; Scale/3.00)',
      'x-dw-client-id':'860f00af-81a2-4e4f-9dda-292b25d3ed94',
      'Cookie':'ak_bmsc=',
    }

    print
    print (url)

    try:
      response=session.get(url,headers=headers,proxies=proxies,timeout=2)
    except KeyboardInterrupt:
      sys.exit(1)
    except:
      print
      print ("Request Error." + " : " + str(datetime.datetime.now()))
      print
      pass

    if "File not found" in response.text:
      print
      print ("File not found.")
      print
      print (str(proxies) + " : " + str(datetime.datetime.now()))
      pass
    elif "Access Denied" in response.text:
      print ("*** ACCESS DENIED ***")
      iteration=iteration+1
      continue

    try:
      productJSON_=json.loads(response.text)
      print
      print (str(proxies) + " : " + str(datetime.datetime.now()))

      print
      for item in productJSON_:
        variants.append([productJSON_[item]['attributes']['size'],productJSON_[item]['id'],int(productJSON_[item]['availability']['ats'])])

        if productJSON_[item]['availability']['inStock']:
          cycleJSON=False
          print (str(item).ljust(10," ") + " : " + " instock     " + " : ATS = "  + str(productJSON_[item]['availability']['ats']).ljust(10," ") + " : IN_STOCK = " + str(productJSON_[item]['availability']['levels']['IN_STOCK']).ljust(10," "))
        else:
          print (str(item).ljust(10," ") + " : "  + " not in stock")

    except KeyboardInterrupt:
      sys.exit(1)
    except:
      cycleJSON=True
      pass

########################################################################

    if cycleJSON:

      variants=[]

      url="https://www.barneys.com/s/BNY/dw/shop/v13_6/products/("+str(productId)+")?expand=availability,bundled_products,images,links,options,prices,promotions,set_products,variations"

      print
      print (url)

      try:
        response=session.get(url,headers=headers,proxies=proxies,timeout=2)
      except KeyboardInterrupt:
        sys.exit(1)
      except:
        print
        print ("Request Error." + " : " + str(datetime.datetime.now()))
        print
        iteration=iteration+1
        continue

      if "File not found" in response.text:
        print
        print ("File not found.")
        pass
      elif "Access Denied" in response.text:
        print ("*** ACCESS DENIED ***")
        iteration=iteration+1
        continue
      else:
        print
        print (str(proxies) + " : " + str(datetime.datetime.now()))

      try:
        productJSON=json.loads(response.text)
        brand=productJSON['data'][0]['brand']
        name=productJSON['data'][0]['name']

        try:
          live=productJSON['data'][0]['c_goLiveDate']
        except KeyboardInterrupt:
          sys.exit(1)
        except:
          live="NULL"

        access=productJSON['data'][0]['c_productAccess']

        try:
          ats=str(productJSON['data'][0]['inventory']['ats'])
          stock_level=str(productJSON['data'][0]['inventory']['stock_level'])
        except KeyboardInterrupt:
          sys.exit(1)
        except:
          ats="-1"
          stock_level="-1"
        cycleJSON=False

        if int(ats) < 1:
          cycleJSON=True

        print
        print (str(productId).ljust(12," ") + " : " + brand + " : " + name + " : " + live + " : " + access)
        for variant in productJSON['data'][0]['variants']:
          if variant['orderable']:
            variants.append([variant['variation_values']['size'],variant['product_id'],1])
          else:
            variants.append([variant['variation_values']['size'],variant['product_id'],0])
          print (variants[-1][0].ljust(12," ") + " : " + variants[-1][1])
        print
        print ("ATS".ljust(12," ") + " : " + ats)
        print ("Stock Level".ljust(12," ") + " : " + stock_level)

      except KeyboardInterrupt:
        sys.exit(1)
      except:
        cycleJSON=True

    if cycleJSON:
      print
      print ("Cycling Product JSON: Sleeping for 1 seconds." + " : " + str(datetime.datetime.now().time()))
      time.sleep(1)

    iteration=iteration+1

########################################################################

  print
  print ("Product is loaded. " + " : " + str(datetime.datetime.now().time()))

  active=False

  while not active:
    iteration=iteration+1
    if variants[iteration%len(variants)][2] > 0:
      session=requests.Session()
      session.mount("http://", requests.adapters.HTTPAdapter(max_retries=1))
      session.mount("https://", requests.adapters.HTTPAdapter(max_retries=1))
      session.verify=False
      session.cookies.clear()
      proxies={
        "http" : "http://"+proxyListCycle[iteration%len(proxyListCycle)],
        "https": "https://"+proxyListCycle[iteration%len(proxyListCycle)]
      }

      headers={
        'User-Agent':'Barneys/12 CFNetwork/758.4.3 Darwin/15.5.0',
        'x-dw-client-id':'860f00af-81a2-4e4f-9dda-292b25d3ed94',
        'Content-Type':'application/json; charset=utf-8',
        'Cookie':'ak_bmsc=',
        'Connection': 'keep-alive',
      }
      url="https://www.barneys.com/s/BNY/dw/shop/v13_6/basket/this/add"
      data={
        "product_id": variants[iteration%len(variants)][1],
        "quantity": 1
      }
      response=session.post(url=url, headers=headers, data=json.dumps(data))
      try:
        cartJSON=json.loads(response.text)
        if cartJSON['product_items'][-1]['quantity'] > 0:
          active=True
      except:
        pass
    else:
      continue

  print
  print ("Product is active. " + " : " + str(datetime.datetime.now().time()))

  elapsed_time = time.time() - start_time

  loop = trollius.get_event_loop()
  loop.run_until_complete(call())

  print
  print "STOP : " + str(datetime.datetime.now())
  print