Merge branch 'save_comments'

pyinstalive/comments.py

@@ -0,0 +1,131 @@
+# -*- coding: utf-8 -*-
+
+import time
+import json
+import codecs
+import sys
+from socket import timeout, error as SocketError
+from ssl import SSLError
+try:
+	# py2
+	from urllib2 import URLError
+	from httplib import HTTPException
+except ImportError:
+	# py3
+	from urllib.error import URLError
+	from http.client import HTTPException
+
+from instagram_private_api import ClientError
+from .logger import log, seperator
+
+"""
+This feature of PyInstaLive was originally written by https://github.com/taengstagram
+The code below and in downloader.py that's related to the comment downloading
+feature is modified by https://github.com/notcammy
+"""
+
+
+class CommentsDownloader(object):
+
+	def __init__(self, api, broadcast, destination_file):
+		self.api = api
+		self.broadcast = broadcast
+		self.destination_file = destination_file
+		self.comments = []
+
+	def get_live(self, first_comment_created_at=0):
+		comments_collected = self.comments
+
+		before_count = len(comments_collected)
+		try:
+			comments_res = self.api.broadcast_comments(
+				self.broadcast['id'], last_comment_ts=first_comment_created_at)
+			comments = comments_res.get('comments', [])
+			first_comment_created_at = (
+				comments[0]['created_at_utc'] if comments else int(time.time() - 5))
+			comments_collected.extend(comments)
+			after_count = len(comments_collected)
+			if after_count > before_count:
+				broadcast = self.broadcast.copy()
+				broadcast.pop('segments', None)     # save space
+				broadcast['comments'] = comments_collected
+				with open(self.destination_file, 'w') as outfile:
+					json.dump(broadcast, outfile, indent=2)
+			self.comments = comments_collected
+
+		except (SSLError, timeout, URLError, HTTPException, SocketError) as e:
+			log('[W] Comment downloading error: %s' % e, "YELLOW")
+		except ClientError as e:
+			if e.code == 500:
+				log('[W] Comment downloading ClientError: %d %s' % (e.code, e.error_response), "YELLOW")
+			elif e.code == 400 and not e.msg:
+				log('[W] Comment downloading ClientError: %d %s' % (e.code, e.error_response), "YELLOW")
+			else:
+				raise e
+		finally:
+			time.sleep(4)
+		return first_comment_created_at
+
+	def get_replay(self):
+		comments_collected = []
+		starting_offset = 0
+		encoding_tag = self.broadcast['encoding_tag']
+		while True:
+			comments_res = self.api.replay_broadcast_comments(
+				self.broadcast['id'], starting_offset=starting_offset, encoding_tag=encoding_tag)
+			starting_offset = comments_res.get('ending_offset', 0)
+			comments = comments_res.get('comments', [])
+			comments_collected.extend(comments)
+			if not comments_res.get('comments') or not starting_offset:
+				break
+			time.sleep(4)
+
+		if comments_collected:
+			self.broadcast['comments'] = comments_collected
+			self.broadcast['initial_buffered_duration'] = 0
+			with open(self.destination_file, 'w') as outfile:
+				json.dump(self.broadcast, outfile, indent=2)
+		self.comments = comments_collected
+
+	def save(self):
+		broadcast = self.broadcast.copy()
+		broadcast.pop('segments', None)
+		broadcast['comments'] = self.comments
+		with open(self.destination_file, 'w') as outfile:
+			json.dump(broadcast, outfile, indent=2)
+
+	@staticmethod
+	def generate_log(comments, download_start_time, srt_file, comments_delay=10.0):
+		python_version = sys.version.split(' ')[0]
+		subtitles_timeline = {}
+		for i, c in enumerate(comments):
+			if 'offset' in c:
+				for k in c['comment'].keys():
+					c[k] = c['comment'][k]
+				c['created_at_utc'] = download_start_time + c['offset']
+			created_at_utc = str(2 * (c['created_at_utc'] // 2))
+			comment_list = subtitles_timeline.get(created_at_utc) or []
+			comment_list.append(c)
+			subtitles_timeline[created_at_utc] = comment_list
+
+		if subtitles_timeline:
+			timestamps = sorted(subtitles_timeline.keys())
+			mememe = False
+			subs = []
+			for i, tc in enumerate(timestamps):
+				t = subtitles_timeline[tc]
+				clip_start = int(tc) - int(download_start_time) + int(comments_delay)
+				if clip_start < 0:
+					clip_start = 0
+
+				srt = ''
+				for c in t:
+						if (c['user']['is_verified']):
+							srt += '{}{}\n\n'.format(time.strftime('%H:%M:%S\n', time.gmtime(clip_start)), '{} {}: {}'.format(c['user']['username'], "(v)", c['text']))
+						else:
+							srt += '{}{}\n\n'.format(time.strftime('%H:%M:%S\n', time.gmtime(clip_start)), '{}: {}'.format(c['user']['username'], c['text']))
+
+				subs.append(srt)
+
+			with codecs.open(srt_file, 'w', 'utf-8-sig') as srt_outfile:
+				srt_outfile.write(''.join(subs))

pyinstalive/downloader.py

@@ -97,11 +97,22 @@ def record_stream(broadcast):
 				log('[W] Could not run file: ' + str(e), "YELLOW")
 
 
+		comment_thread_worker = None
+		if settings.save_comments.title() == "True":
+			try:
+				comments_json_file = settings.save_path + '{}_{}_{}_{}_live_comments.json'.format(settings.current_date, record, broadcast['id'], settings.current_time)
+				comment_thread_worker = threading.Thread(target=get_live_comments, args=(api, broadcast, comments_json_file, dl,))
+				comment_thread_worker.start()
+			except Exception as e:
+				log('[E] An error occurred while checking comments: ' + e, "RED")			
+
+
+
 		dl.run()
 		seperator("GREEN")
 		log('[I] The livestream has ended. (Duration: ' + get_stream_duration(broadcast) + ")", "GREEN")
 		seperator("GREEN")
-		stitch_video(dl, broadcast)
+		stitch_video(dl, broadcast, comment_thread_worker)
 	except KeyboardInterrupt:
 		seperator("GREEN")
 		log('[W] Download has been aborted by the user.', "YELLOW")
@@ -110,7 +121,11 @@ def record_stream(broadcast):
 			dl.stop()
 			stitch_video(dl, broadcast)
 
-def stitch_video(dl, broadcast):
+def stitch_video(dl, broadcast, comment_thread_worker):
+	if comment_thread_worker and comment_thread_worker.is_alive():
+		log("[I] Ending comment saving process...", "GREEN")
+		comment_thread_worker.join()
+
 	if (settings.run_at_finish is not "None"):
 		try:
 			thread = threading.Thread(target=run_script, args=(settings.run_at_finish,))
@@ -218,9 +233,15 @@ def get_replays(user_id):
 						log("[I] Finished downloading replay " + str(current) + " of "  + str(len(broadcasts)) + ".", "GREEN")
 						seperator("GREEN")
 					else:
-						log("[W] No output video file was made, please merge the files manually.", "RED")
-						log("[W] Check if ffmpeg is available by running ffmpeg in your terminal.", "RED")
+						log("[W] No output video file was made, please merge the files manually.", "YELLOW")
+						log("[W] Check if ffmpeg is available by running ffmpeg in your terminal.", "YELLOW")
 						log("", "GREEN")
+
+					if settings.save_comments.title() == "True":
+						log("[I] Checking for available comments to save...", "GREEN")
+						comments_json_file = settings.save_path + '{}_{}_{}_{}_replay_comments.json'.format(settings.current_date, record, broadcast['id'], settings.current_time)
+						get_replay_comments(api, broadcast, comments_json_file, dl)
+
 		log("[I] Finished downloading available replays.", "GREEN")
 		seperator("GREEN")
 		sys.exit(0)

pyinstalive/initialize.py

@@ -26,6 +26,7 @@ def check_ffmpeg():
 
 def check_config_validity(config):
 	try:
+		has_thrown_errors = False
 		settings.username = config.get('pyinstalive', 'username')
 		settings.password = config.get('pyinstalive', 'password')
 
@@ -34,9 +35,11 @@ def check_config_validity(config):
 			if not settings.show_cookie_expiry in bool_values:
 				log("[W] Invalid or missing setting detected for 'show_cookie_expiry', using default value (True)", "YELLOW")
 				settings.show_cookie_expiry = 'true'
+				has_thrown_errors = True
 		except:
 			log("[W] Invalid or missing setting detected for 'show_cookie_expiry', using default value (True)", "YELLOW")
 			settings.show_cookie_expiry = 'true'
+			has_thrown_errors = True
 
 
 
@@ -45,9 +48,11 @@ def check_config_validity(config):
 			if not settings.clear_temp_files in bool_values:
 				log("[W] Invalid or missing setting detected for 'clear_temp_files', using default value (True)", "YELLOW")
 				settings.clear_temp_files = 'true'
+				has_thrown_errors = True
 		except:
 			log("[W] Invalid or missing setting detected for 'clear_temp_files', using default value (True)", "YELLOW")
 			settings.clear_temp_files = 'true'
+			has_thrown_errors = True
 
 
 
@@ -56,9 +61,11 @@ def check_config_validity(config):
 			if not settings.save_replays in bool_values:
 				log("[W] Invalid or missing setting detected for 'save_replays', using default value (True)", "YELLOW")
 				settings.save_replays = 'true'
+				has_thrown_errors = True
 		except:
 			log("[W] Invalid or missing setting detected for 'save_replays', using default value (True)", "YELLOW")
 			settings.save_replays = 'true'
+			has_thrown_errors = True
 
 
 
@@ -68,15 +75,18 @@ def check_config_validity(config):
 				if not os.path.isfile(settings.run_at_start):
 					log("[W] Path to file given for 'run_at_start' does not exist, using default value (None)", "YELLOW")
 					settings.run_at_start = "None"
+					has_thrown_errors = True
 				else:
 					if not settings.run_at_start.split('.')[-1] == 'py':
 						log("[W] File given for 'run_at_start' is not a Python script, using default value (None)", "YELLOW")
 						settings.run_at_start = "None"
+						has_thrown_errors = True
 			else:
 				settings.run_at_start = "None"
 		except:
 			log("[W] Invalid or missing settings detected for 'run_at_start', using default value (None)", "YELLOW")
 			settings.run_at_start = "None"
+			has_thrown_errors = True
 
 
 
@@ -86,18 +96,36 @@ def check_config_validity(config):
 				if not os.path.isfile(settings.run_at_finish):
 					log("[W] Path to file given for 'run_at_finish' does not exist, using default value (None)", "YELLOW")
 					settings.run_at_finish = "None"
+					has_thrown_errors = True
 				else:
 					if not settings.run_at_finish.split('.')[-1] == 'py':
 						log("[W] File given for 'run_at_finish' is not a Python script, using default value (None)", "YELLOW")
 						settings.run_at_finish = "None"
+						has_thrown_errors = True
 			else:
 				settings.run_at_finish = "None"
 
 		except:
 			log("[W] Invalid or missing settings detected for 'run_at_finish', using default value (None)", "YELLOW")
 			settings.run_at_finish = "None"
+			has_thrown_errors = True
 
 
+		try:
+			settings.save_comments = config.get('pyinstalive', 'save_comments').title()
+			if sys.version.split(' ')[0].startswith('2') and settings.save_comments == "True":
+				log("[W] Comment saving is not supported in Python 2 and will be ignored.", "YELLOW")
+				settings.save_comments = 'false'
+				has_thrown_errors = True
+			else:
+				if not settings.show_cookie_expiry in bool_values:
+					log("[W] Invalid or missing setting detected for 'save_comments', using default value (False)", "YELLOW")
+					settings.save_comments = 'false'
+					has_thrown_errors = True
+		except:
+			log("[W] Invalid or missing setting detected for 'save_comments', using default value (False)", "YELLOW")
+			settings.save_comments = 'false'
+			has_thrown_errors = True
 
 		try:
 			settings.save_path = config.get('pyinstalive', 'save_path')
@@ -107,12 +135,17 @@ def check_config_validity(config):
 			else:
 				log("[W] Invalid or missing setting detected for 'save_path', falling back to path: " + os.getcwd(), "YELLOW")
 				settings.save_path = os.getcwd()
+				has_thrown_errors = True
 
 			if not settings.save_path.endswith('/'):
 				settings.save_path = settings.save_path + '/'
 		except:
 			log("[W] Invalid or missing setting detected for 'save_path', falling back to path: " + os.getcwd(), "YELLOW")
 			settings.save_path = os.getcwd()
+			has_thrown_errors = True
+
+		if has_thrown_errors:
+			seperator("GREEN")
 
 		if not (len(settings.username) > 0):
 			log("[E] Invalid or missing setting detected for 'username'.", "RED")