Prevent --clean from deleting ongoing stream folders and some more fixes

.gitignore

@@ -34,3 +34,5 @@ snippets\.py
 downloaded/
 
 requirements\.txt
+
+*.lock

pyinstalive/comments.py

@@ -4,6 +4,7 @@ import codecs
 import json
 import sys
 import time
+import os
 
 from socket import error as SocketError
 from socket import timeout
@@ -102,6 +103,9 @@ class CommentsDownloader(object):
 
 	@staticmethod
 	def generate_log(comments, download_start_time, log_file, comments_delay=10.0):
+		comment_log_save_path = os.path.dirname(os.path.dirname(log_file))
+		comment_log_file_name = os.path.basename(log_file)
+		log_file = os.path.join(comment_log_save_path, comment_log_file_name)
 		python_version = sys.version.split(' ')[0]
 		subtitles_timeline = {}
 		wide_build = sys.maxunicode > 65536

pyinstalive/downloader.py

@@ -122,6 +122,7 @@ def download_livestream(broadcast):
 		print_status(False)
 		log('[I] MPD URL     : {:s}'.format(mpd_url), "GREEN")
 		seperator("GREEN")
+		open(os.path.join(output_dir,'folder.lock'), 'a').close()
 		log('[I] Downloading livestream... press [CTRL+C] to abort.', "GREEN")
 
 		if (settings.run_at_start is not "None"):
@@ -137,7 +138,7 @@ def download_livestream(broadcast):
 		comment_thread_worker = None
 		if settings.save_comments.title() == "True":
 			try:
-				comments_json_file = settings.save_path + '{}_{}_{}_{}_live_comments.json'.format(settings.current_date, user_to_record, broadcast.get('id'), settings.current_time)
+				comments_json_file = os.path.join(output_dir, '{}_{}_{}_{}_live_comments.json'.format(settings.current_date, user_to_record, broadcast.get('id'), settings.current_time))
 				comment_thread_worker = threading.Thread(target=get_live_comments, args=(instagram_api, broadcast, comments_json_file, broadcast_downloader,))
 				comment_thread_worker.start()
 			except Exception as e:
@@ -156,14 +157,19 @@ def download_livestream(broadcast):
 			stitch_video(broadcast_downloader, broadcast, comment_thread_worker)
 	except Exception as e:
 		log("[E] Could not download livestream: {:s}".format(str(e)), "RED")
+		try:
+		    os.remove(os.path.join(output_dir,'folder.lock'))
+		except OSError:
+		    pass
 
 
 def stitch_video(broadcast_downloader, broadcast, comment_thread_worker):
 	try:
 
 		live_mp4_file = settings.save_path + '{}_{}_{}_{}_live.mp4'.format(settings.current_date, user_to_record, broadcast.get('id'), settings.current_time)
-		live_json_file = settings.save_path + '{}_{}_{}_{}_live_comments.json'.format(settings.current_date, user_to_record, broadcast.get('id'), settings.current_time)
-		live_comments_file = live_json_file.replace(".json", ".log")
+		live_folder_path = live_mp4_file.split('.mp4')[0] + "_downloads"
+		live_json_file = os.path.join(settings.save_path, live_folder_path, '{}_{}_{}_{}_live_comments.json'.format(settings.current_date, user_to_record, broadcast.get('id'), settings.current_time))
+		live_comments_file = live_mp4_file.replace(".mp4", "_live_comments.log")
 
 		live_files = [live_mp4_file]
 
@@ -183,17 +189,19 @@ def stitch_video(broadcast_downloader, broadcast, comment_thread_worker):
 				log('[W] Could not execute command: {:s}'.format(str(e)), "YELLOW")
 
 		log('[I] Stitching downloaded files into video...', "GREEN")		
-
 		try:
 			if settings.clear_temp_files.title() == "True":
 				broadcast_downloader.stitch(live_mp4_file, cleartempfiles=True)
 			else:
 				broadcast_downloader.stitch(live_mp4_file, cleartempfiles=False)
 			log('[I] Successfully stitched downloaded files into video.', "GREEN")
+			try:
+			    os.remove(os.path.join(live_folder_path,'folder.lock'))
+			except OSError:
+			    pass
 			if settings.clear_temp_files.title() == "True":
-				live_folder_to_del = live_mp4_file.split('.mp4')[0] + "_downloads"
 				try:
-					shutil.rmtree(live_folder_to_del)
+					shutil.rmtree(live_folder_path)
 				except Exception as e:
 					log("[E] Could not remove temp folder: {:s}".format(str(e)), "RED")
 			if settings.ftp_enabled:
@@ -207,14 +215,26 @@ def stitch_video(broadcast_downloader, broadcast, comment_thread_worker):
 		except ValueError as e:
 			log('[E] Could not stitch downloaded files: {:s}\n[E] Likely the download duration was too short and no temp files were saved.'.format(str(e)), "RED")
 			seperator("GREEN")
+			try:
+			    os.remove(os.path.join(live_folder_path,'folder.lock'))
+			except OSError:
+			    pass
 			sys.exit(1)
 		except Exception as e:
 			log('[E] Could not stitch downloaded files: {:s}'.format(str(e)), "RED")
 			seperator("GREEN")
+			try:
+			    os.remove(os.path.join(live_folder_path,'folder.lock'))
+			except OSError:
+			    pass
 			sys.exit(1)
 	except KeyboardInterrupt:
 			log('[I] Aborted stitching process, no video was created.', "YELLOW")
 			seperator("GREEN")
+			try:
+			    os.remove(os.path.join(live_folder_path,'folder.lock'))
+			except OSError:
+			    pass
 			sys.exit(0)
 
 
@@ -297,9 +317,9 @@ def download_replays(broadcasts):
 					mpd=broadcast.get('dash_manifest'),
 					output_dir=output_dir,
 					user_agent=instagram_api.user_agent)
-
+				open(os.path.join(output_dir,'folder.lock'), 'a').close()
 				replay_mp4_file = settings.save_path + '{}_{}_{}_{}_replay.mp4'.format(settings.current_date, user_to_record, broadcast.get('id'), settings.current_time)
-				replay_json_file = settings.save_path + '{}_{}_{}_{}_replay_comments.json'.format(settings.current_date, user_to_record, broadcast.get('id'), settings.current_time)
+				replay_json_file = os.path.join(output_dir, '{}_{}_{}_{}_replay_comments.json'.format(settings.current_date, user_to_record, broadcast.get('id'), settings.current_time))
 				replay_comments_file = replay_json_file.replace(".json", ".log")
 
 				replay_files = [replay_mp4_file]
@@ -316,6 +336,10 @@ def download_replays(broadcasts):
 
 				if (len(replay_saved) == 1):
 					log("[I] Finished downloading replay {:s} of {:s}.".format(str(current), str(len(broadcasts))), "GREEN")
+					try:
+					    os.remove(os.path.join(output_dir,'folder.lock'))
+					except OSError:
+					    pass
 					if settings.ftp_enabled:
 						try:
 							upload_ftp_files(replay_files)
@@ -337,6 +361,10 @@ def download_replays(broadcasts):
 	except Exception as e:
 		log('[E] Could not save replay: {:s}'.format(str(e)), "RED")
 		seperator("GREEN")
+		try:
+		    os.remove(os.path.join(output_dir,'folder.lock'))
+		except OSError:
+		    pass
 		sys.exit(1)
 	except KeyboardInterrupt:
 		seperator("GREEN")

pyinstalive/initialize.py

@@ -260,8 +260,9 @@ def new_config():
 
 def clean_download_dir():
 	dir_delcount = 0
-	file_delcount = 0
 	error_count = 0
+	lock_count = 0
+
 	log('[I] Cleaning up temporary files and folders...', "GREEN")
 	try:
 		if sys.version.split(' ')[0].startswith('2'):
@@ -273,39 +274,34 @@ def clean_download_dir():
 
 		for directory in directories:
 			if directory.endswith('_downloads'):
-				try:
-					shutil.rmtree(settings.save_path + directory)
-					dir_delcount += 1
-				except Exception as e:
-					log("[E] Could not remove temp folder: {:s}".format(str(e)), "RED")
-					error_count += 1
-		for file in files:
-			if file.endswith('_comments.json'):
-				try:
-					os.remove(settings.save_path + file)
-					file_delcount += 1
-				except Exception as e:
-					log("[E] Could not remove temp file: {:s}".format(str(e)), "RED")
-					error_count += 1
+				if not any(filename.endswith('.lock') for filename in os.listdir(settings.save_path + directory)):
+					try:
+						shutil.rmtree(settings.save_path + directory)
+						dir_delcount += 1
+					except Exception as e:
+						log("[E] Could not remove temp folder: {:s}".format(str(e)), "RED")
+						error_count += 1
+				else:
+					lock_count += 1
 		seperator("GREEN")
 		log('[I] The cleanup has finished.', "YELLOW")
-		if dir_delcount == 0 and file_delcount == 0 and error_count == 0:
-			log('[I] No files or folders were removed.', "YELLOW")
+		if dir_delcount == 0 and error_count == 0 and lock_count == 0:
+			log('[I] No folders were removed.', "YELLOW")
 			seperator("GREEN")
 			return
 		log('[I] Folders removed:     {:d}'.format(dir_delcount), "YELLOW")
-		log('[I] Files removed:       {:d}'.format(file_delcount), "YELLOW")
+		log('[I] Locked folders:      {:d}'.format(lock_count), "YELLOW")
 		log('[I] Errors:              {:d}'.format(error_count), "YELLOW")
 		seperator("GREEN")
 	except KeyboardInterrupt as e:
 		seperator("GREEN")
 		log("[W] The cleanup has been aborted.", "YELLOW")
-		if dir_delcount == 0 and file_delcount == 0 and error_count == 0:
-			log('[I] No files or folders were removed.', "YELLOW")
+		if dir_delcount == 0 and error_count == 0 and lock_count == 0:
+			log('[I] No folders were removed.', "YELLOW")
 			seperator("GREEN")
 			return
 		log('[I] Folders removed:     {:d}'.format(dir_delcount), "YELLOW")
-		log('[I] Files removed:       {:d}'.format(file_delcount), "YELLOW")
+		log('[I] Locked folders:      {:d}'.format(lock_count), "YELLOW")
 		log('[I] Errors:              {:d}'.format(error_count), "YELLOW")
 		seperator("GREEN")