download twits and parse the ouput as json with nodejs

Author: carlos

nodejsutilities/aws/express_twitter/app.js

@@ -0,0 +1,19 @@
+var app = require('express').createServer(),
+    twitter = require('ntwitter');
+
+app.listen(3000);
+
+var twit = new twitter({
+  consumer_key: 'aaaaaaadsfasdf',
+  consumer_secret: 'adsfasdfañlkjasfdlkjasdf',
+  access_token_key: 'adsfasdfasdff',
+  access_token_secret: 'asdfadsfasdfasdfasdf'
+});
+
+//twit.stream('statuses/filter', { track: ['love', 'hate'] }, function(stream) {
+twit.stream('statuses/filter', { track: ['bbva'] }, function(stream) {
+  stream.on('data', function (data) {
+    console.log(JSON.stringify(data));
+    console.log(",")
+  });
+});

nodejsutilities/aws/express_twitter/format_logs.sh

@@ -0,0 +1,3 @@
+echo "["
+sed "s/^{ created_at/@\'created_at\'/" $1 | sed 's/\[Object\]/null/g'   | grep -v "retweeted_status" |  sed 's/"//g' | sed 's/\s\s*\([A-Za-z_][A-Za-z_]*\):/"\1":/' | sed 's/^@/,\{/' | sed "s/'/\"/g" 
+echo "]"

nodejsutilities/aws/express_twitter/logs0.txt

@@ -0,0 +1,7 @@
+[{"created_at":"Sun Oct 06 17:34:36 +0000 2013","id":386907356371894300,"id_str":"386907356371894272","text":"RT @_Futbolero_: Don Marc Bartra es el jugador con más precisión en el pase de toda la Liga BBVA, con un porcentaje de 95,3%. #Crack","source":"<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1426988508,"id_str":"1426988508","name":"Chisco José","screen_name":"QueOndaMucha","location":"","url":null,"description":"el fútbol es para mi lo que el amor es para muchos.","protected":false,"followers_count":76,"friends_count":75,"listed_count":0,"created_at":"Tue May 14 04:35:46 +0000 2013","favourites_count":639,"utc_offset":-18000,"time_zone":"Central Time (US & Canada)","geo_enabled":true,"verified":false,"statuses_count":952,"lang":"es","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http://a0.twimg.com/profile_background_images/378800000017449805/d90c5c50fbbce4a84537f86fe1223e6e.jpeg","profile_background_image_url_https":"https://si0.twimg.com/profile_background_images/378800000017449805/d90c5c50fbbce4a84537f86fe1223e6e.jpeg","profile_background_tile":true,"profile_image_url":"http://a0.twimg.com/profile_images/378800000378025820/151ee70985f0e5f35370c2ff325380f4_normal.jpeg","profile_image_url_https":"https://si0.twimg.com/profile_images/378800000378025820/151ee70985f0e5f35370c2ff325380f4_normal.jpeg","profile_banner_url":"https://pbs.twimg.com/profile_banners/1426988508/1372092017","profile_link_color":"0084B4","profile_sidebar_border_color":"FFFFFF","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweeted_status":{"created_at":"Sun Oct 06 17:12:51 +0000 2013","id":386901880409841660,"id_str":"386901880409841664","text":"Don Marc Bartra es el jugador con más precisión en el pase de toda la Liga BBVA, con un porcentaje de 95,3%. #Crack","source":"web","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":167604665,"id_str":"167604665","name":"-1899-","screen_name":"_Futbolero_","location":"España","url":"http://goo.gl/K2pFYa","description":"[Pablo, 1984] Valenciano que da su punto de vista sobre el mundo del fútbol. (Aunque a veces sea muy crítico) #Donamédula","protected":false,"followers_count":28565,"friends_count":148,"listed_count":387,"created_at":"Sat Jul 17 01:05:52 +0000 2010","favourites_count":31,"utc_offset":7200,"time_zone":"Madrid","geo_enabled":false,"verified":false,"statuses_count":52978,"lang":"es","contributors_enabled":false,"is_translator":false,"profile_background_color":"F7A751","profile_background_image_url":"http://a0.twimg.com/profile_background_images/658415718/q5prbzwwfu494d29rff4.jpeg","profile_background_image_url_https":"https://si0.twimg.com/profile_background_images/658415718/q5prbzwwfu494d29rff4.jpeg","profile_background_tile":false,"profile_image_url":"http://a0.twimg.com/profile_images/378800000552460638/32ca756a4115ae3344eb09b413e4bea8_normal.jpeg","profile_image_url_https":"https://si0.twimg.com/profile_images/378800000552460638/32ca756a4115ae3344eb09b413e4bea8_normal.jpeg","profile_banner_url":"https://pbs.twimg.com/profile_banners/167604665/1380986509","profile_link_color":"0084B4","profile_sidebar_border_color":"FFFFFF","profile_sidebar_fill_color":"000501","profile_text_color":"FF0522","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":82,"favorite_count":23,"entities":{"hashtags":[{"text":"Crack","indices":[109,115]}],"symbols":[],"urls":[],"user_mentions":[]},"favorited":false,"retweeted":false,"lang":"es"},"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"Crack","indices":[126,132]}],"symbols":[],"urls":[],"user_mentions":[{"screen_name":"_Futbolero_","name":"-1899-","id":167604665,"id_str":"167604665","indices":[3,15]}]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"es"}
+,
+{"created_at":"Sun Oct 06 17:34:47 +0000 2013","id":386907400793759740,"id_str":"386907400793759744","text":"RT @Todomercadoweb: Celta, Luis Enrique: \"El Atlético es el rival más poderoso al que nos hemos enfrentado\" http://t.co/u1Ols4yrTu","source":"<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":1739065286,"id_str":"1739065286","name":"JM","screen_name":"feelblondnine","location":"Granada","url":null,"description":"Atleti       Vivo di mezzo chilometro in mezzo chilometro,niente piú conta per me...       Fernando Torres sempre nel mio cuore","protected":false,"followers_count":44,"friends_count":264,"listed_count":0,"created_at":"Sat Sep 07 01:13:31 +0000 2013","favourites_count":254,"utc_offset":null,"time_zone":null,"geo_enabled":false,"verified":false,"statuses_count":486,"lang":"es","contributors_enabled":false,"is_translator":false,"profile_background_color":"C0DEED","profile_background_image_url":"http://abs.twimg.com/images/themes/theme1/bg.png","profile_background_image_url_https":"https://abs.twimg.com/images/themes/theme1/bg.png","profile_background_tile":false,"profile_image_url":"http://a0.twimg.com/profile_images/378800000422671885/2921e92fd4379dbd216f1b77678ec962_normal.jpeg","profile_image_url_https":"https://si0.twimg.com/profile_images/378800000422671885/2921e92fd4379dbd216f1b77678ec962_normal.jpeg","profile_banner_url":"https://pbs.twimg.com/profile_banners/1739065286/1378817679","profile_link_color":"0084B4","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":true,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweeted_status":{"created_at":"Sun Oct 06 14:06:15 +0000 2013","id":386854920387051500,"id_str":"386854920387051520","text":"Celta, Luis Enrique: \"El Atlético es el rival más poderoso al que nos hemos enfrentado\" http://t.co/u1Ols4yrTu","source":"<a href=\"http://www.todomercadoweb.es\" rel=\"nofollow\">Todo Mercado Web APP</a>","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":529498317,"id_str":"529498317","name":"Todomercadoweb","screen_name":"Todomercadoweb","location":"","url":"http://www.todomercadoweb.es","description":"El sitio del mercado futbolístico actualizado 24 horas al día.","protected":false,"followers_count":63102,"friends_count":321,"listed_count":378,"created_at":"Mon Mar 19 16:09:39 +0000 2012","favourites_count":6,"utc_offset":10800,"time_zone":"Athens","geo_enabled":true,"verified":false,"statuses_count":28628,"lang":"it","contributors_enabled":false,"is_translator":false,"profile_background_color":"008000","profile_background_image_url":"http://a0.twimg.com/profile_background_images/452814100/twitter2.png","profile_background_image_url_https":"https://si0.twimg.com/profile_background_images/452814100/twitter2.png","profile_background_tile":false,"profile_image_url":"http://a0.twimg.com/profile_images/2083494771/n_normal.jpg","profile_image_url_https":"https://si0.twimg.com/profile_images/2083494771/n_normal.jpg","profile_link_color":"008000","profile_sidebar_border_color":"C0DEED","profile_sidebar_fill_color":"DDEEF6","profile_text_color":"333333","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":1,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[{"url":"http://t.co/u1Ols4yrTu","expanded_url":"http://www.todomercadoweb.es/liga-bbva/celta-luis-enrique-el-atletico-es-el-rival-mas-poderoso-al-que-nos-hemos-enfrentado-59376","display_url":"todomercadoweb.es/liga-bbva/celt…","indices":[88,110]}],"user_mentions":[]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"lang":"es"},"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[],"symbols":[],"urls":[{"url":"http://t.co/u1Ols4yrTu","expanded_url":"http://www.todomercadoweb.es/liga-bbva/celta-luis-enrique-el-atletico-es-el-rival-mas-poderoso-al-que-nos-hemos-enfrentado-59376","display_url":"todomercadoweb.es/liga-bbva/celt…","indices":[108,130]}],"user_mentions":[{"screen_name":"Todomercadoweb","name":"Todomercadoweb","id":529498317,"id_str":"529498317","indices":[3,18]}]},"favorited":false,"retweeted":false,"possibly_sensitive":false,"filter_level":"medium","lang":"es"}
+,
+{"created_at":"Sun Oct 06 17:35:11 +0000 2013","id":386907503021531140,"id_str":"386907503021531137","text":"\"Marc Bartra es el jugador con más precisión en el pase de toda la Liga BBVA [...]\" yo también sé pasarsela a Valdés #valors @LorenzoRguez","source":"<a href=\"http://twitter.com/download/android\" rel=\"nofollow\">Twitter for Android</a>","truncated":false,"in_reply_to_status_id":null,"in_reply_to_status_id_str":null,"in_reply_to_user_id":null,"in_reply_to_user_id_str":null,"in_reply_to_screen_name":null,"user":{"id":197078267,"id_str":"197078267","name":"Enrique Delgado ☢","screen_name":"enrikedelg","location":"Sevilla, Santiago d Compostela","url":null,"description":"Licenciado en Matemáticas destinado al lluvioso norte, al que le gusta la yesca; y medalla de bronce ☢ pinfloyd ☢ en las fiestas de La Cumbre 2013","protected":false,"followers_count":137,"friends_count":160,"listed_count":1,"created_at":"Thu Sep 30 16:07:24 +0000 2010","favourites_count":2,"utc_offset":7200,"time_zone":"Madrid","geo_enabled":true,"verified":false,"statuses_count":1676,"lang":"es","contributors_enabled":false,"is_translator":false,"profile_background_color":"1A1B1F","profile_background_image_url":"http://abs.twimg.com/images/themes/theme9/bg.gif","profile_background_image_url_https":"https://abs.twimg.com/images/themes/theme9/bg.gif","profile_background_tile":false,"profile_image_url":"http://a0.twimg.com/profile_images/2903676946/dcc4714e672ef199815ea603e2eaffa3_normal.png","profile_image_url_https":"https://si0.twimg.com/profile_images/2903676946/dcc4714e672ef199815ea603e2eaffa3_normal.png","profile_banner_url":"https://pbs.twimg.com/profile_banners/197078267/1355311646","profile_link_color":"2FC2EF","profile_sidebar_border_color":"000000","profile_sidebar_fill_color":"252429","profile_text_color":"666666","profile_use_background_image":true,"default_profile":false,"default_profile_image":false,"following":null,"follow_request_sent":null,"notifications":null},"geo":null,"coordinates":null,"place":null,"contributors":null,"retweet_count":0,"favorite_count":0,"entities":{"hashtags":[{"text":"valors","indices":[117,124]}],"symbols":[],"urls":[],"user_mentions":[{"screen_name":"LorenzoRguez","name":"Lorenzo Rodríguez","id":332309597,"id_str":"332309597","indices":[125,138]}]},"favorited":false,"retweeted":false,"filter_level":"medium","lang":"es"}
+]
+

nodejsutilities/aws/express_twitter/package.json

@@ -0,0 +1,9 @@
+{
+  "name":"socket.io-twitter-example",
+  "version":"0.0.1",
+  "private":true,
+  "dependencies":{
+  	"express":"2.5.4",
+  	"ntwitter":"0.2.10"
+  }
+}

nodejsutilities/aws/express_twitter/readlogs.js

@@ -0,0 +1,38 @@
+function isoStringToDate(s) {
+  try{
+  	var b = s.split(/[ :+]/ig);
+  	return parseInt(b[8]+""+b[3]+""+b[2]);
+  }catch(e){ return null; }
+}
+
+var fs = require('fs') ,AWS = require('aws-sdk'),roclas = require('../roclas.js');
+AWS.config.loadFromPath('../config.json');
+AWS.config.update({region: 'eu-west-1'});
+
+var dynamodb = new AWS.DynamoDB();
+
+fs.readFile('./logs1.txt', 'utf8', function (err,data) { 
+	if (err) { console.log('Error: ' + err); return; }
+	a= JSON.parse(data); 
+	for(i in a){ 
+		a[i]["created_at"]=isoStringToDate(a[i]["created_at"]);
+  		try{ a[i]["user_id"]=a[i]["user"]["id"];
+  		}catch(e){a[i]["user_id"]=0;}
+		var item={};
+		//text=roclas.findInObject(a[i],"text")[0];
+		//item[text]=a[i][text];
+		for(f in a[i]){
+		 if(typeof(a[i][f])=="object"){
+			//item[f]={"S":JSON.stringify(a[i][f])}; 
+			//console.log("typeof "+f+" == object");
+		 }
+		 else{item[f]={"S":""+a[i][f]}; }
+		}
+		item.user_id={"N":""+a[i]["user_id"]};
+		item.created_at={"N":""+a[i]["created_at"]};
+		dynamodb.putItem({TableName:'tweets',Item:item },function(err, data){
+    			if (err) { console.log("Error: "+err); 
+    			} else { console.log(data); }
+		});
+	}
+});

nodejsutilities/parse_json_lines.js

@@ -0,0 +1,20 @@
+if(process.argv.length<3){
+	console.error("missing argument (file to parse)");
+	return(-1);
+}
+
+var fs = require('fs'),readline = require('readline');
+
+var rd = readline.createInterface({
+    input: fs.createReadStream(process.argv[2]),
+    output: process.stdout,
+    terminal: false
+});
+
+rd.on('line', function(line) {
+    try{
+	var l=JSON.parse(line);
+	console.log(JSON.stringify(l,null,4));
+    }catch(error){}
+    
+});