brikis98 · November 2, 2011 03:57 · brikis98 · Nov 2, 2011 · dajobe · Nov 3, 2011
diff --git a/Average page views by industry b/Average page views by industry
 pv_by_industry = GROUP profile_view by viewee_industry_id
 pv_avg_by_industry = FOREACH pv_by_industry 
  GENERATE group as viewee_industry_id, AVG(profie_view) AS average_pv; 
diff --git a/Count profile views per member b/Count profile views per member
 pv_group_by_viewee = GROUP profile_view by viewee_member_id
 pv_with_count = FOREACH pv_group_by_viewee {
  GENERATE viewee_member_id, COUNT_STAR(profile_view) as pv_count;
 }
diff --git a/Find members with more profile views b/Find members with more profile views
 connections_pv_source = JOIN pv_with_count BY viewee_member_id, member_connections BY source_member_id;
 connections_pv_dest = JOIN pv_with_count BY viewee_member_id, 
 connections_pv_source BY dest_member_id;
 connections_with_higher_vp = FILTER connections_pv_dest BY connections_pv_dest::pv_count > connections_pv_source::pv_count;
diff --git a/gistfile1.txt b/gistfile1.txt
 urls = load 'dataset' AS (url, category, pagerank);
 groups = GROUP urls by category;
 bigGroup = FILTER groups by COUNT(groups) > 100000;
 STORE result bigGroup INTO 'bigGroupOutput';
diff --git a/Join all data together b/Join all data together
 few_pv_email_data =  JOIN connections_with_higher_pv by viewee_industry_id, pv_avg_by_industry by viewee_industry_id;
diff --git a/Member_Connections Table b/Member_Connections Table
 # Member_Connections Table

 +---------------------------------+------------------+
 | Field                           | Type             | 
 +---------------------------------+------------------+
 | source_member_id                | int              |
 | dest_member_id                  | int              |
 +---------------------------------+------------------+
diff --git a/Profile_View Table b/Profile_View Table
 # Profile_View Table

 +---------------------------------+------------------+
 | Field                           | Type             | 
 +---------------------------------+------------------+
 | viewee_member_id                | int              |
 | viewer_member_id                | int              |
 | viewee_industry_id              | int              |
 | tracking_time                   | timestamp        |
 +---------------------------------+------------------+
	pv_by_industry = GROUP profile_view by viewee_industry_id
	pv_avg_by_industry = FOREACH pv_by_industry
	GENERATE group as viewee_industry_id, AVG(profie_view) AS average_pv;
	pv_group_by_viewee = GROUP profile_view by viewee_member_id
	pv_with_count = FOREACH pv_group_by_viewee {
	GENERATE viewee_member_id, COUNT_STAR(profile_view) as pv_count;
	}
	connections_pv_source = JOIN pv_with_count BY viewee_member_id, member_connections BY source_member_id;
	connections_pv_dest = JOIN pv_with_count BY viewee_member_id,
	connections_pv_source BY dest_member_id;
	connections_with_higher_vp = FILTER connections_pv_dest BY connections_pv_dest::pv_count > connections_pv_source::pv_count;
	urls = load 'dataset' AS (url, category, pagerank);
	groups = GROUP urls by category;
	bigGroup = FILTER groups by COUNT(groups) > 100000;
	STORE result bigGroup INTO 'bigGroupOutput';
	# Member_Connections Table

	+---------------------------------+------------------+
	\| Field \| Type \|
	+---------------------------------+------------------+
	\| source_member_id \| int \|
	\| dest_member_id \| int \|
	+---------------------------------+------------------+
	# Profile_View Table

	+---------------------------------+------------------+
	\| Field \| Type \|
	+---------------------------------+------------------+
	\| viewee_member_id \| int \|
	\| viewer_member_id \| int \|
	\| viewee_industry_id \| int \|
	\| tracking_time \| timestamp \|
	+---------------------------------+------------------+