In LBBO/netflix-migrate#31 is was discussed, that setting viewing history was not possible based on the known APIs. Since that means, that migrating multiple profiles to the same account is not possible, I took a shot and created a working variant based on what was already given.
netflix2.js
// TODO: use ES6 import / export
// import async from 'async'
// import cheerio from 'cheerio'
// import extend from 'extend'
// import request from 'request'
// import { sprintf } from 'sprintf-js'
// import util from 'util'
// import vm from 'vm'
// import HttpError from './httpError'
// import constants from './constants'
// import manifest from '../package'
const cheerio = require('cheerio')
const extend = require('extend')
const request = require('request-promise-native')
const { sprintf } = require('sprintf-js')
const vm = require('vm')
const HttpError = require('./httpError')
const errorLogger = require('./errorLogger')
const constants = require('./constants')
const fetch = require('node-fetch')
const fs = require('fs').promises
const path = require('path')
/** @namespace */
class Netflix {
/**
* Creates a new Netlfix API library instance
*
* @constructor
* @param {Object} options
*/
constructor(options) {
console.warn('Using new Netflix2 class!')
const cookieJar = request.jar()
options = extend(true, {
cookieJar
}, options)
this.cookieJar = options.cookieJar
this.netflixContext = {}
this.endpointIdentifiers = {}
this.authUrls = {}
this.activeProfile = null
}
async __request(url, options = {}) {
console.log(url);
const config = {
redirect: 'follow',
...options,
headers: {
cookie: this.cookie,
...options.headers
},
}
return await fetch(url, config)
}
/**
* Sets cookies, API endpoints, and the authURL that must be used to
* make API calls
*
* This must be called before using any other functions
*
* @param {{email: string, password: string, cookies: string}} credentials
*
*/
async login(credentials) {
try {
this.cookie = credentials.cookies
if (credentials) {
// const loginForm = await this.__getLoginForm(credentials)
// await this.__postLoginForm(loginForm)
await this.__getContextDataFromUrls([constants.yourAccountUrl, constants.manageProfilesUrl])
console.log('Login successful!')
} else {
// Try using cookies from previous login instead of credentials
await this.__getContextDataFromUrls([constants.yourAccountUrl, constants.manageProfilesUrl])
console.log('Welcome back!')
}
} catch (err) {
errorLogger(err)
// This error will need to be handled by the callee as in
// netflix.login().then().catch((err) => handleError(err))
// Otherwise, it will always throw an "UnhandledPromiseRejectionWarning".
throw new Error('Something went wrong. For more information, see previous log statements.')
}
}
/**
* Browse movies, to simply get all films use Category ID 34399
*
* @param {number} genreId The Netflix Category ID, Like https://www.netflix.com/browse/genre/34399
* @param {number} page The content is paged, this is the page number.
* @param {number} perPage How many items do you want per page?
*
* @returns {Promise<Object>} movies
*/
async browse(genreId, page, perPage) {
const pager = {
from: page * perPage,
to: (page + 1) * perPage
}
const defaultQuery = ['genres', genreId, 'su']
// The Netflix shakti API is a bit strange,
// It needs a path structured in this way.
const options = {
method: 'POST',
body: {
paths: [
[...defaultQuery, pager, 'title', 'genres'],
[...defaultQuery, pager, 'boxarts', '_342x192', 'jpg']
]
}
}
const endpoint = constants.pathEvaluatorEndpointUrl
const response = await this.__apiRequest(endpoint, options)
return await response.json()
}
/**
* @typedef {Object} Profile
* @property {string} firstName
* @property {string} rawFirstName
* @property {string} guid
* @property {boolean} isAccountOwner
* @property {boolean} isActive
* @property {boolean} defaultKidsProfile
* @property {string} experience
* @property {boolean} isAutoCreated
* @property {string} avatarName
* @property {{32: string, 50: string, 64: string, 80: string, 100: string, 112: string, 160: string, 200: string,
* 320: string, }} avatarImages
* @property {boolean} canEdit
* @property {boolean} isDefault
*/
/**
* @returns {Profile[]} profiles
*/
async getProfiles() {
const options = {}
const endpoint = constants.profilesEndpointUrl
try {
const response = await this.__apiRequest(endpoint, options)
const body = await response.json()
if (response.status !== 200) {
throw new HttpError(response.status, response.statusText)
}
// TODO; check if status is 2xx
return body.profiles
} catch (err) {
console.error(err)
}
}
/**
*
* @param {string} guid - can be found from {}
*/
async switchProfile(guid) {
try {
const endpoint = `${constants.switchProfileEndpointUrl}?switchProfileGuid=${guid}`
console.log(endpoint);
const response = await this.__apiRequest(endpoint)
const body = await response.json()
if (false) {
throw new Error('There was an error while trying to switch profile')
} else {
this.activeProfile = guid
await this.__getContextDataFromUrls([constants.yourAccountUrl, constants.manageProfilesUrl])
}
} catch (err) {
errorLogger(err)
throw new Error("Couldn't switch profiles. For more information, see previous log statements.")
}
}
/**
*
* @typedef {Object} rating
* @property {"thumb"|"star"} ratingType
* @property {string} title
* @property {number} movieID
* @property {number} rating
* @property {string} date
* @property {number} timestamp
* @property {number} comparableDate
*/
/**
*
* @returns {Promise<rating[]>}
*/
async getRatingHistory() {
let ratingItems = []
let page = 0
let pages = 1
while (page < pages) {
const json = await this.__getRatingHistory(page)
page = json.page + 1
pages = Math.floor(json.totalRatings / json.size) + 1
ratingItems = ratingItems.concat(json.ratingItems)
}
return ratingItems
}
/**
* @typedef {Object} viewingHistoryItem
* @property {string} title
* @property {string} videoTitle
* @property {number} movieID
* @property {string} country
* @property {number} bookmark - Amount of seconds the user has already seen
* @property {number} duration - Total duration of episode/movie in seconds
* @property {number} date
* @property {number} deviceType
* @property {string} dateStr
* @property {number} index
* @property {string} topNodeId
* @property {string} rating
* @property {number} series
* @property {string} seriesTitle
* @property {string} seasonDescriptor
* @property {string} episodeTitle
*/
/**
* Downloads the whole list of viewed movies.
* The Netflix endpoint is paged.
* This structure is copied from getRatingHistory.
*
* @returns viewingHistoryItem[]
*/
async getViewingHistory() {
let viewedItems = []
let page = 0
let pages = 1
while (page < pages) {
const json = await this.__getViewingHistory(page)
page = json.page + 1
pages = Math.floor(json.vhSize / json.size) + 1
viewedItems = viewedItems.concat(json.viewedItems)
}
return viewedItems
}
/**
* Hides viewing history for a specific movie or episode
* @param {number} movieID - the ID of the movie (e.g. 80057281 for "Stranger Things")
*/
async hideSingleEpisodeFromViewingHistory(movieID) {
return await this.__hideSpecificViewingHistory(movieID, false)
}
/**
* Hides viewing history for a the whole series with the supplied movieID
* @param {number} movieID - the ID of the movie (e.g. 80057281 for "Stranger Things")
*/
async hideEntireSeriesFromViewingHistory(movieID) {
return await this.__hideSpecificViewingHistory(movieID, true)
}
/**
*
* @param {number} movieID
* @param {boolean} seriesAll
*/
async __hideSpecificViewingHistory(movieID, seriesAll) {
const options = {
body: {
movieID: movieID,
seriesAll: seriesAll,
authURL: this.authUrls[constants.yourAccountUrl]
},
method: 'POST'
}
const endpoint = constants.viewingActivity
const response = await this.__apiRequest(endpoint, options)
return await response.json()
}
/**
* Hides ALL viewing history: this may not always reset the viewing history per series.
* Use hideMovieViewingHistory passing the movieID and setting seriesAll to true
* to reset that series' history back to the first episode
*/
async hideAllViewingHistory() {
return await this.__hideAllViewingHistory()
}
async __hideAllViewingHistory() {
const options = {
body: {
hideAll: true,
authURL: this.authUrls[constants.yourAccountUrl]
},
method: 'POST'
}
const endpoint = constants.viewingActivity
const response = await this.__apiRequest(endpoint, options)
return await response.json()
}
/**
*
* @param {number} page
* @returns {Object}
*/
async __getViewingHistory(page) {
const endpoint = `${constants.viewingActivity}?pg=${page}`
try {
const response = await this.__apiRequest(endpoint)
return await response.json()
} catch (err) {
errorLogger(err)
throw new Error("Couldn't get your viewing history. For more information, see previous log statements.")
}
}
/**
*
* @param {boolean} isThumbRating
* @param {number} titleId
* @param {number} rating
*/
async __setRating(isThumbRating, titleId, rating) {
const endpoint = isThumbRating ? constants.setThumbRatingEndpointUrl : constants.setVideoRatindEndpointUrl
const options = {
body: JSON.stringify({
rating: rating,
authURL: this.authUrls[constants.yourAccountUrl],
// Note the capital I in titleId in the if-case vs. the lower case i in the else-case. This is necessary
// due to the Shakti API.
[isThumbRating ? 'titleId' : 'titleid']: titleId,
}),
headers: {
'content-type': 'application/json',
},
method: 'POST',
}
try {
const response = await this.__apiRequest(endpoint, options)
const body = await response.json()
if (body.newRating !== rating) {
throw new Error('Something went wrong! The saved rating does not match the rating that was supposed to be saved.')
}
} catch (err) {
errorLogger(err)
throw new Error(`Couldn't set ${isThumbRating ? 'thumb rating' : 'star rating'}. For more information, see previous log statements.`)
}
}
/**
*
* @param {number} titleId
* @param {number} rating
*/
async setStarRating(titleId, rating) {
await this.__setRating(false, titleId, rating)
}
/**
*
* @param {number} titleId
* @param {number} rating
*/
async setThumbRating(titleId, rating) {
await this.__setRating(true, titleId, rating)
}
/**
*
* @returns {Promise<Profile>}
*/
async getActiveProfile() {
const endpoint = constants.profilesEndpointUrl
const options = {}
const response = await this.__apiRequest(endpoint, options)
const body = await response.json()
return body.active
}
getAvatarUrl(avatarName, size) {
return sprintf(constants.avatarUrl, size || 320, avatarName.split('icon')[1])
}
async setAvatar(avatarName) {
const endpoint = constants.pathEvaluatorEndpointUrl
const options = {
body: {
callPath: ['profiles', this.activeProfile, 'edit'],
params: [null, null, null, avatarName, null],
authURL: this.authUrls[constants.manageProfilesUrl]
},
method: 'POST'
}
const response = await this.__apiRequest(`${endpoint}?method=call`, options)
return await response.json()
}
/**
*
* @param {{email: string, password: string}} credentials
* @returns {Object}
*/
async __getLoginForm(credentials) {
try {
const response = await this.__request(constants.baseUrl + constants.loginUrl)
// When the statusCode is 403, that means we have been trying to login too many times in succession with incorrect credentials.
if (response.status === 403) {
throw new Error('Your credentials are either incorrect or you have attempted to login too many times.')
} else if (response.status !== 200) {
throw new HttpError(response.status, response.statusText)
} else {
const $ = cheerio.load(await response.text())
let form = $('.login-input-email')
.parent('form')
.serializeArray()
// reduce array of key-value pairs to object
.reduce((obj, pair) => {
obj[pair.name] = pair.value
return obj
}, {})
form.userLoginId = credentials.email
form.password = credentials.password
return form
}
} catch (err) {
errorLogger(err)
throw new Error('Could not retrieve login page. For more information, see previous log statements.')
}
}
/**
*
* @param {Object} form
*/
async __postLoginForm(form) {
const options = {
method: 'POST',
form: form,
}
try {
const response = await this.__request(constants.baseUrl + constants.loginUrl, options)
if (response.statusCode !== 302) {
// we expect a 302 redirect upon success
const $ = cheerio.load(await response.text())
// This will always get the correct error message that is displayed on the Netflix website.
const message = $('.ui-message-contents', '.hybrid-login-form-main').text()
throw new Error(message)
}
} catch (err) {
errorLogger(err)
throw new Error('Check your credentials. For more information, see previous log statements.')
}
}
/**
*
* @param {number} page
*/
async __getRatingHistory(page) {
const endpoint = `${constants.ratingHistoryEndpointUrl}?pg=${page}`
try {
const response = await this.__apiRequest(endpoint)
return await response.json()
} catch (err) {
errorLogger(err)
throw new Error('There was something wrong getting your rating history. For more information, see previous log statements.')
}
}
/**
*
* @param {string} endpoint
* @param {Object} options
* @returns {Object}
*/
async __apiRequest(endpoint, options) {
try {
const response = await this.__request(this.apiRoot + endpoint, options)
if (response.status !== 200) {
throw new HttpError(response.status, response.statusText)
} else {
return response
}
} catch (err) {
errorLogger(err)
throw new Error('There was something wrong with your request. For more information, see previous log statements.')
}
}
/**
*
* @param {string} url
*/
async __getContextData(url) {
let body
try {
const response = await this.__request(constants.baseUrl + url)
if (response.status !== 200) {
throw new HttpError(response.status, response.statusText)
} else {
const context = {
window: {},
netflix: {}
}
vm.createContext(context)
body = await response.text()
const $ = cheerio.load(body)
$('script').map((index, element) => {
// don't run external scripts
if (!element.attribs.src) {
const script = $(element).text()
if (script.indexOf('window.netflix') === 0) {
vm.runInContext(script, context)
}
}
})
const shaktiApiRootUrl = 'https://www.netflix.com/api/shakti/'
/*
* For some reason, context.netflix.reactContext does not always exist. The cause may be wether an account is active
* or not, but that is not for sure.
*
* Currently, this fixes the issue, as the shakti API root seems to always be identical and endpointIdentifiers
* seem to always be an empty object. This is a quick and dirty fix due to lack of more information!
*/
if (context.netflix.reactContext) {
this.netflixContext = context.netflix.reactContext.models
this.apiRoot = shaktiApiRootUrl + "mre";
this.endpointIdentifiers = this.netflixContext.serverDefs.data.endpointIdentifiers
if (!context.netflix.reactContext.models.truths.data['CURRENT_MEMBER']) {
throw new Error('Inactive account')
}
if (!context.netflix.reactContext.models.memberContext) {
throw new Error('You need to setup a profile.')
}
this.authUrls[url] = context.netflix.reactContext.models.memberContext.data.userInfo.authURL
} else if (context.netflix.contextData) {
this.netflixContext = context.netflix.contextData
this.apiRoot = shaktiApiRootUrl + context.netflix.contextData.serverDefs.BUILD_IDENTIFIER
this.endpointIdentifiers = {}
// TODO: The auth URL is probably somewhere else. Figure out where it is exactly when a user logs into an inactive
// account.
this.authUrls[url] = context.netflix.contextData.authURL
} else {
throw new Error(
'An error occurred that appears to be similar to ' +
'https://github.com/LBBO/netflix-migrate/issues/24 !'
)
}
}
} catch (err) {
errorLogger(err)
if (body) {
const filePath = path.join(process.cwd(), 'errorResponsePage.html')
await fs.writeFile(filePath, body)
console.error(`The exact response HTML file was saved to ${filePath}`)
}
throw new Error('There was a problem fetching user data. For more information, see previous log statements.')
}
}
/**
*
* @param {...string} urls
*/
async __getContextDataFromUrls(urls) {
for (const url of urls) {
await this.__getContextData(url)
}
}
}
// TODO: use ES6 import / export
// export default Netflix
module.exports = Netflix
proxify-patch
diff --git a/README.md b/README.md
index 6a7f1bb..71d7ca7 100644
--- a/README.md
+++ b/README.md
@@ -80,6 +80,8 @@ FILTER:
-resp-fd, -response-dsl string[] Response Filter DSL
-req-mrd, -request-match-replace-dsl string[] Request Match-Replace DSL
-resp-mrd, -response-match-replace-dsl string[] Response Match-Replace DSL
+ -resp-head-add, -response-header-add string[] Responseheader Add
+ -resp-head-set, -response-header-set string[] Responseheader Set
NETWORK:
-ha, -http-addr string Listening HTTP IP and Port address (ip:port) (default "127.0.0.1:8888")
diff --git a/cmd/mitmrelay/mitmrelay.go b/cmd/mitmrelay/mitmrelay.go
index b7f34f4..857d12d 100644
--- a/cmd/mitmrelay/mitmrelay.go
+++ b/cmd/mitmrelay/mitmrelay.go
@@ -115,6 +115,8 @@ func main() {
proxyOpts.HTTPProxy = options.HTTPProxy
proxyOpts.RequestMatchReplaceDSL = []string{options.RequestMatchReplaceDSL}
proxyOpts.ResponseMatchReplaceDSL = []string{options.ResponseMatchReplaceDSL}
+ proxyOpts.ResponseHeaderAdd = []string{options.ResponseHeaderAdd}
+ proxyOpts.ResponseHeaderSet = []string{options.ResponseHeaderSet}
if options.Timeout >= 0 {
proxyOpts.Timeout = time.Duration(options.Timeout) * time.Second
diff --git a/internal/runner/options.go b/internal/runner/options.go
index 1792f2f..11e9029 100644
--- a/internal/runner/options.go
+++ b/internal/runner/options.go
@@ -32,6 +32,8 @@ type Options struct {
RequestMatchReplaceDSL goflags.StringSlice // Request Match-Replace DSL
ResponseDSL goflags.StringSlice // Response Filter DSL
ResponseMatchReplaceDSL goflags.StringSlice // Request Match-Replace DSL
+ ResponseHeaderAdd goflags.StringSlice // Responseheader Add
+ ResponseHeaderSet goflags.StringSlice // Responseheader Set
UpstreamHTTPProxies goflags.StringSlice // Upstream HTTP comma separated Proxies (e.g. http://proxyip:proxyport)
UpstreamSocks5Proxies goflags.StringSlice // Upstream SOCKS5 comma separated Proxies (e.g. socks5://proxyip:proxyport)
UpstreamProxyRequestsNumber int // Number of requests before switching upstream proxy
@@ -79,6 +81,8 @@ func ParseOptions() *Options {
flagSet.StringSliceVarP(&options.ResponseDSL, "response-dsl", "resp-fd", nil, "Response Filter DSL", goflags.StringSliceOptions),
flagSet.StringSliceVarP(&options.RequestMatchReplaceDSL, "request-match-replace-dsl", "req-mrd", nil, "Request Match-Replace DSL", goflags.StringSliceOptions),
flagSet.StringSliceVarP(&options.ResponseMatchReplaceDSL, "response-match-replace-dsl", "resp-mrd", nil, "Response Match-Replace DSL", goflags.StringSliceOptions),
+ flagSet.StringSliceVarP(&options.ResponseHeaderAdd, "response-header-add", "resp-head-add", nil, "Responseheader Add", goflags.StringSliceOptions),
+ flagSet.StringSliceVarP(&options.ResponseHeaderSet, "response-header-set", "resp-head-set", nil, "Responseheader Set", goflags.StringSliceOptions),
)
flagSet.CreateGroup("network", "Network",
diff --git a/internal/runner/runner.go b/internal/runner/runner.go
index bee2bf0..72b444f 100644
--- a/internal/runner/runner.go
+++ b/internal/runner/runner.go
@@ -50,6 +50,8 @@ func NewRunner(options *Options) (*Runner, error) {
DNSFallbackResolver: options.DNSFallbackResolver,
RequestMatchReplaceDSL: options.RequestMatchReplaceDSL,
ResponseMatchReplaceDSL: options.ResponseMatchReplaceDSL,
+ ResponseHeaderAdd: options.ResponseHeaderAdd,
+ ResponseHeaderSet: options.ResponseHeaderSet,
DumpRequest: options.DumpRequest,
DumpResponse: options.DumpResponse,
OutputJsonl: options.OutputJsonl,
diff --git a/pkg/types/userdata.go b/pkg/types/userdata.go
index bd2df74..ec9a7e3 100644
--- a/pkg/types/userdata.go
+++ b/pkg/types/userdata.go
@@ -4,6 +4,7 @@ type UserData struct {
ID string
Match bool
HasResponse bool
+ NeedsClose bool
Host string
}
diff --git a/proxy.go b/proxy.go
index c9d3987..bb71065 100644
--- a/proxy.go
+++ b/proxy.go
@@ -14,6 +14,7 @@ import (
"strconv"
"strings"
+ "github.com/elazarl/goproxy/transport"
"github.com/armon/go-socks5"
"github.com/haxii/fastproxy/bufiopool"
"github.com/haxii/fastproxy/superproxy"
@@ -57,6 +58,8 @@ type Options struct {
DNSFallbackResolver string
RequestMatchReplaceDSL []string
ResponseMatchReplaceDSL []string
+ ResponseHeaderAdd []string
+ ResponseHeaderSet []string
OnRequestCallback OnRequestFunc
OnResponseCallback OnResponseFunc
Deny []string
@@ -82,13 +85,39 @@ type Proxy struct {
// ModifyRequest
func (p *Proxy) ModifyRequest(req *http.Request) error {
+ var needsClose bool
+
+ if req.URL.Path == "/netflix-migrate-iframe" {
+ req.Method = http.MethodGet
+ req.Header = map[string][]string{}
+ req.Body = io.NopCloser(strings.NewReader(""))
+ req.URL, _ = url.Parse("http://localhost/netflix/import.html")
+ req.Host = "localhost"
+ needsClose = true
+ } else if req.URL.Path == "/netflixData.js" {
+ req.Method = http.MethodGet
+ req.Header = map[string][]string{}
+ req.Body = io.NopCloser(strings.NewReader(""))
+ req.URL, _ = url.Parse("http://localhost/netflix/netflixData.js")
+ req.Host = "localhost"
+ needsClose = true
+ }
+
ctx := martian.NewContext(req)
// disable upgrading http connections to https by default
- ctx.Session().MarkInsecure()
+
+ if req.Host == "localhost" {
+ ctx.Session().MarkInsecure()
+ } else {
+ ctx.Session().MarkSecure()
+ req.URL.Scheme = "https"
+ }
+
// setup passthrought and hijack here
userData := types.UserData{
ID: ctx.ID(),
Host: req.Host,
+ NeedsClose: needsClose,
}
// If callbacks are given use them (for library use cases)
@@ -151,9 +180,9 @@ func (p *Proxy) ModifyResponse(resp *http.Response) error {
}
}
userData.Match = matchStatus
- // perform match and replace
- if len(p.options.ResponseMatchReplaceDSL) != 0 {
- _ = p.MatchReplaceResponse(resp)
+ // perform match and replace and/or additions
+ if len(p.options.ResponseMatchReplaceDSL) != 0 || len(p.options.ResponseHeaderAdd) != 0 || len(p.options.ResponseHeaderSet) != 0 {
+ _ = p.MatchReplaceAddResponse(resp)
}
_ = p.logger.LogResponse(resp, *userData)
if resp.StatusCode == 301 || resp.StatusCode == 302 {
@@ -167,6 +196,11 @@ func (p *Proxy) ModifyResponse(resp *http.Response) error {
}
resp.Close = true
}
+
+ if userData.NeedsClose {
+ resp.Close = true
+ }
+
return nil
}
@@ -204,11 +238,12 @@ func (p *Proxy) MatchReplaceRequest(req *http.Request) error {
req.Header = requestNew.Header
req.Body = requestNew.Body
req.URL = requestNew.URL
+
return nil
}
-// MatchReplaceRequest strings or regex
-func (p *Proxy) MatchReplaceResponse(resp *http.Response) error {
+// MatchReplaceAddRequest strings or regex
+func (p *Proxy) MatchReplaceAddResponse(resp *http.Response) error {
// Set Content-Length to zero to allow automatic calculation
resp.ContentLength = 0
@@ -238,11 +273,38 @@ func (p *Proxy) MatchReplaceResponse(resp *http.Response) error {
return err
}
+ for _, expr := range p.options.ResponseHeaderAdd {
+ ar := strings.Split(expr, ":")
+ if len(ar) != 2 {
+ return fmt.Errorf("cannot add header, expected length 2, got length %d", len(ar))
+ }
+
+ responseNew.Header.Add(ar[0], ar[1])
+ }
+
+ for _, expr := range p.options.ResponseHeaderSet {
+ ar := strings.Split(expr, ":")
+ if len(ar) != 2 {
+ return fmt.Errorf("cannot set header, expected length 2, got length %d", len(ar))
+ }
+
+ responseNew.Header.Set(ar[0], ar[1])
+ }
+
// closes old body to allow memory reuse
resp.Body.Close()
resp.Header = responseNew.Header
resp.Body = responseNew.Body
resp.ContentLength = responseNew.ContentLength
+
+ if location, ok := resp.Header["Location"]; ok {
+ for loci, locv := range location {
+ location[loci] = strings.Replace(locv, "http:", "https:", -1)
+ }
+
+ resp.Header["Location"] = location
+ }
+
return nil
}
@@ -258,11 +320,25 @@ func (p *Proxy) Run() error {
// http proxy
if p.httpProxy != nil {
p.httpProxy.TLSPassthroughFunc = func(req *http.Request) bool {
- // if !stringsutil.ContainsAny(req.URL.Host, "avatars") {
- // log.Printf("Skipped MITM for %v", req.URL.Host)
- // return true
- // }
- return false
+ mitmHosts := []string{
+ "geolocation.onetrust.com",
+ "netflix.com",
+ }
+
+ skip := true
+ for _, host := range mitmHosts {
+ if strings.Contains(req.URL.Host, host) {
+ skip = false
+ break
+ }
+ }
+
+ if !skip {
+ return false
+ }
+
+ log.Printf("Skipped MITM for %v", req.URL.Host)
+ return true
}
p.httpProxy.SetRequestModifier(p)
@@ -348,14 +424,7 @@ func (p *Proxy) setupHTTPProxy() error {
// getRoundTripper returns RoundTripper configured with options
func (p *Proxy) getRoundTripper() (http.RoundTripper, error) {
- roundtrip := &http.Transport{
- MaxIdleConnsPerHost: -1,
- MaxIdleConns: 0,
- MaxConnsPerHost: 0,
- TLSClientConfig: &tls.Config{
- InsecureSkipVerify: true,
- },
- }
+ roundtrip := &http.Transport{Proxy: transport.ProxyFromEnvironment}
if len(p.options.UpstreamHTTPProxies) > 0 {
roundtrip = &http.Transport{Proxy: func(req *http.Request) (*url.URL, error) {
import.html
<html>
<head>
<script src="netflixData.js"></script>
<script>
var id, title, item, timestamp, progress, movies, w;
var currentMovieNumber = 1;
const watchSeconds = 20;
const watchBeforeEndSeconds = 180;
async function watchMovie (movie){
id = movie.movieID
title = movie.title
item = currentMovieNumber.toString().padStart(movies.length.toString().length, 0)
timestamp = new Date().toLocaleTimeString()
progress = `${item}/${movies.length}`
const url = `https://www.netflix.com/watch/${id}`
document.title = `${progress} Netflix Views`
w.src = url;
checkIframeLoaded(url, 0);
}
async function watchMovies() {
movies = _netflixData.viewingHistory.reverse();
w = document.getElementById("iframe");
watchMovie(movies[0]);
}
function status(msg, url){
console.log(msg, url)
var el = document.querySelector('.content')
el.innerHTML = `<p>${msg} <a href="${url}">${url}</a></p>` + el.innerHTML
}
async function waitForPlayer(iframe, url, cnt) {
if(cnt < 40) {
try {
let videoPlayer = iframe.contentWindow.window.netflix.appContext.state.playerApp.getAPI().videoPlayer;
let player = videoPlayer.getVideoPlayerBySessionId(videoPlayer.getAllPlayerSessionIds()[0]);
if (typeof player === "undefined") {
throw new Error("player undefined");
}
if (!player.getReady()) {
throw new Error("player not ready");
}
getDurationAndPlay(player, url, 0);
return
} catch (e) {
setTimeout(function() {waitForPlayer(iframe, url, cnt+1)}, 500);
}
} else {
watchNextMovie(url, false);
}
}
async function waitForURL(iframe, url, cnt) {
if(cnt < 40) {
if (iframe.contentWindow.location.href == url) {
waitForPlayer(iframe, url, 0);
return
} else if (iframe.contentWindow.location.href.includes("origId=" + id)) {
watchNextMovie(url, false);
} else {
setTimeout(function() {waitForURL(iframe, url, cnt+1)}, 500);
}
} else {
watchNextMovie(url, false);
}
}
async function getDurationAndPlay(player, url, cnt) {
if(cnt < 40) {
try {
let duration = player.getDuration();
if (duration == 0) {
throw new Error("duration is still 0")
}
let watchToSeconds = duration/1000;
try {
player.play();
setTimeout(function() {seek(player, watchToSeconds-watchBeforeEndSeconds, url, 0)}, 2000);
} catch (e) {
watchNextMovie(url, false);
}
return
} catch (e) {
setTimeout(function() {getDurationAndPlay(player, url, cnt+1)}, 500);
}
} else {
watchNextMovie(url, false);
}
}
async function seek(player, startAt, url, cnt) {
if(cnt < 40) {
try {
try {
player.seek(startAt*1000)
setTimeout(function() {watchNextMovie(url, true, startAt+watchSeconds, watchSeconds)}, watchSeconds * 1000);
} catch (e) {
watchNextMovie(url, false);
}
return
} catch (e) {
setTimeout(function() {seek(player, url, cnt+1)}, 500);
}
} else {
watchNextMovie(url, false);
}
}
async function checkIframeLoaded(url, cnt) {
if(cnt < 100) {
// Get a handle to the iframe element
let iframe = document.getElementById('iframe');
let iframeDoc = iframe.contentDocument || iframe.contentWindow.document;
// Check if loading is complete
if ( iframeDoc.readyState === 'complete' && iframe.contentWindow.location.href != "about:blank") {
waitForURL(iframe, url, 0);
return
} else {
setTimeout(function() {checkIframeLoaded(url, cnt+1)}, 100);
}
} else {
watchNextMovie(url, false);
}
}
async function watchNextMovie(url, ok, watchToSeconds) {
let msg = "";
if (typeof watchToSeconds !== "undefined") {
msg = `${progress}: ${title} for ${watchSeconds} seconds beginning at ${Math.floor((watchToSeconds-watchSeconds)/60)}:${((Math.floor((watchToSeconds%60)) < 10)?'0':'')}${Math.floor((watchToSeconds%60))} (current time: ${timestamp})`
} else {
msg = `${progress}: ${title} (current time: ${timestamp})`
}
if (ok === true) {
status(msg + " (OK)", url)
} else {
status(msg + " (FAIL)", url)
}
currentMovieNumber = currentMovieNumber + 1;
if (movies.length >= currentMovieNumber) {
console.log("1");
watchMovie(movies[currentMovieNumber-1]);
}
}
window.onload = watchMovies
</script>
</head>
<body>
<H2>Netflix Importer</H2>
<table style="width:100%">
<tr>
<td style="width:75%"><div class="content"></div></td>
<td style="width:25%"><iframe is="x-frame-bypass" id="iframe" allow="encrypted-media *;" style="width:100%;height:800px" /></td>
</tr>
</table>
</body>
</html>
-
Use netflix-migrate to get the history. You need to replace netflix2.js however after npm install (node_modules/netflix2/lib/netflix2.js) due to changed shakti API.
1.1. This assumes, that you've set the values as envvars:
node index.js --cookie "NetflixId=$NETFLIX_ID;SecureNetflixId=$SECURE_NETFLIX_ID" --profile $PROFILE_NAME --$OPERATION $FILE_NAME -
Get proxify (I'm using WSL) - go 1.19 is neccessary to be installed manually (e.g. via https://github.com/udhos/update-golang) here: https://github.com/projectdiscovery/proxify
2.1. Check out commit b377e8b54ef8439d4aff2a29d7a23e8c60c63612 (just to myke sure my patch works).
2.2. Apply patch (I have created custom flags to match the specific needs here)
2.3. cd into cmd/proxify ,
go buildit2.4. start via command
./proxify -config=/home/wsl/.config/proxify -resp-mrd "replace(response, 'x-frame-options', '')" -resp-mrd "replace(response, 'X-Frame-Options', '')" -resp-mrd "replace(response, 'Strict', 'None')" -resp-mrd "replace(response, 'Lax', 'None')" -resp-mrd "replace(response, 'Strict', 'none')" -resp-mrd "replace(response, 'lax', 'None')" -resp-mrd "replace(response, '; Expires=', '; SameSite=None; Secure; Expires=')" -resp-mrd "replace(response, 'Access-Control-Allow-Origin: https://www.netflix.com', 'Access-Control-Allow-Origin: \*')" -resp-head-set "Access-Control-Allow-Origin: *"2.5. hint: if you access specific netflix URLs such as the viewing history when using proxify, netflix wont work anymore. Do
rm -rf logsin the appropriate folder to fix that. -
install apache and place the import.html and
netflixData.jsinto the subfoldernetflixinside your served folder (e.g. /var/www/html/netflix).netflixData.jsis your exportData but at the beginning of the contentlet _netflixData =is prepended.3.1. make sure you can access your apache from your host via http://localhost
-
set your proxy (I suggest you use firefox) and import the cacert.pem into the trusted CAs inside firefox (proxify will act as MITM) from /home/wsl/.config/proxify
-
Access https://www.netflix.com, log in and choose the target profile. This will determine, which profile the data is imported to!
-
Access https://www.netflix.com/netflix-migrate-iframe and let it run. There will be some failures, as netflix might not have all the stuff that you watched years ago anymore. This is expected! Basically the script automates netflix watching in an iframe. I start 180 seconds from the end and watch 20 seconds, just to make sure the viewing history gets set properly.
Proxify is needed as MITM, since we would not be able to control netflix inside an iframe otherwise. There are several security measured in place, that normally prevent this. Since I hardcoded www.netflix.com, it will not work without using www..
Feel free to adjust this script, it seemed to work just fine for four profiles now :)