refactor api

.gitignore

@@ -1,2 +1,4 @@
 index.html
 vvvvidb.sqlite3
+
+__pycache__/

api.py

@@ -0,0 +1,113 @@
+import requests
+
+class Api():
+	ua = "Mozilla/5.0 (Windows; U; Win98; en-US; rv:0.9.4.2) Gecko/20020502 CS 2000 7.0/7.0"
+
+	def __init__(self):
+		print("Staring VVVVID Api")
+
+	def ds(self, h):
+		g = "MNOPIJKL89+/4567UVWXQRSTEFGHABCDcdefYZabstuvopqr0123wxyzklmnghij"
+
+		def f(m):
+			l = []
+			o = 0
+			b = False
+			m_len = len(m)
+			while ((not b) and o < m_len):
+				n = m[o] << 2
+				o += 1
+				k = -1
+				j = -1
+				if o < m_len:
+					n += m[o] >> 4
+					o += 1
+					if o < m_len:
+						k = (m[o - 1] << 4) & 255
+						k += m[o] >> 2
+						o += 1
+						if o < m_len:
+							j = (m[o - 1] << 6) & 255
+							j += m[o]
+							o += 1
+						else:
+							b = True
+					else:
+						b = True
+				else:
+					b = True
+				l.append(n)
+				if k != -1:
+					l.append(k)
+				if j != -1:
+					l.append(j)
+			return l
+
+		c = []
+		for e in h:
+			c.append(g.index(e))
+
+		c_len = len(c)
+		for e in range(c_len * 2 - 1, -1, -1):
+			a = c[e % c_len] ^ c[(e + 1) % c_len]
+			c[e % c_len] = a
+
+		c = f(c)
+		d = ''
+		for e in c:
+			d += chr(e)
+
+		return d
+
+	def get_settings(self):
+		settings = requests.get('https://www.vvvvid.it/vvvvid/settings', headers={'User-Agent': Api.ua})
+		if settings.status_code != 200:
+			return None
+		self.stream_url = settings.json()['data']['defaultStreamingServer']
+		return self.stream_url
+
+	def login(self):
+		login = requests.get('https://www.vvvvid.it/user/login', headers={'User-Agent': Api.ua})
+		self.conn_id = login.json()['data']['conn_id']
+
+	def get_info(self, show_id):
+		info = requests.get('https://www.vvvvid.it/vvvvid/ondemand/' + str(show_id) + '/info/?conn_id=' + self.conn_id, headers={'User-Agent': Api.ua})
+		info.encoding = 'utf-8'
+		if info.json()['result'] == 'ok':
+			return info.json()['data']
+		else:
+			return False
+
+	def get_seasons(self, show_id):
+		seasons = requests.get('https://www.vvvvid.it/vvvvid/ondemand/' + str(show_id) + '/seasons/?conn_id=' + self.conn_id, headers={'User-Agent': Api.ua})
+		if seasons.json()['result'] == 'ok' and seasons.json()['data'] and seasons.json()['data'][0]['episodes']:
+			return seasons.json()['data']
+		else:
+			return []
+
+	def get_episodes(self, season_id, show_id):
+		episodes = requests.get('https://www.vvvvid.it/vvvvid/ondemand/' + str(show_id) + '/season/' + str(season_id) + '?conn_id=' + self.conn_id, headers={'User-Agent': Api.ua}).json()
+		if episodes['result'] == 'ok' and episodes['data'] and episodes['data'][0]['embed_info']:
+			return episodes['data']
+		else:
+			return []
+
+	def format_episodes(self, season_id, show_id, episodes):
+		eps = []
+		count = 0
+		for k in episodes:
+			count += 1
+			if k['embed_info']:
+				if k['video_type'] == 'video/rcs':
+					embed_info = self.ds(k['embed_info'])
+					embed_info = 'https' + embed_info[4:30] + 'i' + embed_info[31:-12] + 'master.m3u8'
+				elif k['video_type'] == 'video/vvvvid':
+					embed_info = 'https' + self.stream_url[4:] + self.ds(k['embed_info']) + '/playlist.m3u8'
+				elif k['video_type'] == 'video/youtube':
+					embed_info = self.ds(k['embed_info'])
+				elif k['video_type'] == 'video/kenc':
+					embed_info = self.ds(k['embed_info'])
+				else:
+					embed_info = self.ds(k['embed_info'])
+			eps.append((count, show_id, season_id, k['video_type'], embed_info))
+		return eps

db.py

@@ -0,0 +1,83 @@
+import os
+import sqlite3
+
+class VVVVIDatabase():
+
+	def __init__(self, path):
+		self.path = path
+		self.is_valid = os.path.isfile(self.path)
+
+	def create(self):
+		con = sqlite3.connect(self.path)
+		cur = con.cursor()
+		cur.execute("CREATE TABLE series (id INTEGER, name TEXT NOT NULL, season_id INTEGER, type TEXT, PRIMARY KEY (id, season_id));")
+		cur.execute("CREATE TABLE episodes (id INTEGER, serie_id INTEGER, season_id INTEGER, cdn_url TEXT NOT NULL, type TEXT NOT NULL);")
+		con.commit()
+		con.close()
+
+	def last_serie(self):
+		con = sqlite3.connect(self.path)
+		cur = con.cursor()
+		cur.execute("SELECT id FROM series ORDER BY id DESC LIMIT 1;")
+		rows = cur.fetchall()
+		last = (rows[0][0] + 1) if len(rows) > 0 else 0
+		con.commit()
+		con.close()
+		return last
+
+	def series_id(self):
+		con = sqlite3.connect(self.path)
+		cur = con.cursor()
+		cur.execute("SELECT id, season_id FROM series ORDER BY id, season_id DESC;")
+		rows = cur.fetchall()
+		con.commit()
+		con.close()
+		# getting series in a useful format for later
+		series = {}
+		for i,s in rows:
+			if series.get(i) is None:
+				series[i] = []
+			series[i].append(s)
+		return series
+
+	def series_episodes(self, serie_id):
+		con = sqlite3.connect(self.path)
+		cur = con.cursor()
+		cur.execute("SELECT id, serie_id, season_id FROM episodes ORDER BY id, serie_id, season_id DESC;")
+		rows = cur.fetchall()
+		con.commit()
+		con.close()
+		# getting series in a useful format for later
+		series = {}
+		for i,s in rows:
+			if series.get(i) is None:
+				series[i] = []
+			series[i].append(s)
+		return series		
+
+
+	def insert_serie(self, serie):
+		con = sqlite3.connect(self.path)
+		cur = con.cursor()
+		try:
+			cur.execute("INSERT INTO series (id, name, season_id, type) VALUES (?, ?, ?, ?);", serie)
+			con.commit()
+			con.close()
+			return True
+		except sqlite3.IntegrityError:
+			# serie gia' presente
+			pass
+		return False
+
+	def insert_episodes(self, eps):
+		con = sqlite3.connect(self.path)
+		cur = con.cursor()
+		try:
+			cur.executemany("INSERT INTO episodes (id, serie_id, season_id, type, cdn_url) VALUES (?, ?, ?, ?, ?);", eps)
+			con.commit()
+			con.close()
+			return True
+		except sqlite3.IntegrityError:
+			# episodi gia' presenti
+			pass
+		return False

vvvvget.py

@@ -2,168 +2,87 @@ import os
 import json
 import requests
 import sqlite3
+from db import VVVVIDatabase
+from api import Api
 
-ua = "Mozilla/5.0 (Windows; U; Win98; en-US; rv:0.9.4.2) Gecko/20020502 CS 2000 7.0/7.0"
+api = Api()
-
+stream_url = api.get_settings()
-def ds(h):
+if stream_url is None:
-	g = "MNOPIJKL89+/4567UVWXQRSTEFGHABCDcdefYZabstuvopqr0123wxyzklmnghij"
-
-	def f(m):
-		l = []
-		o = 0
-		b = False
-		m_len = len(m)
-		while ((not b) and o < m_len):
-			n = m[o] << 2
-			o += 1
-			k = -1
-			j = -1
-			if o < m_len:
-				n += m[o] >> 4
-				o += 1
-				if o < m_len:
-					k = (m[o - 1] << 4) & 255
-					k += m[o] >> 2
-					o += 1
-					if o < m_len:
-						j = (m[o - 1] << 6) & 255
-						j += m[o]
-						o += 1
-					else:
-						b = True
-				else:
-					b = True
-			else:
-				b = True
-			l.append(n)
-			if k != -1:
-				l.append(k)
-			if j != -1:
-				l.append(j)
-		return l
-
-	c = []
-	for e in h:
-		c.append(g.index(e))
-
-	c_len = len(c)
-	for e in range(c_len * 2 - 1, -1, -1):
-		a = c[e % c_len] ^ c[(e + 1) % c_len]
-		c[e % c_len] = a
-
-	c = f(c)
-	d = ''
-	for e in c:
-		d += chr(e)
-
-	return d
-
-def get_settings():
-	settings = requests.get('https://www.vvvvid.it/vvvvid/settings', headers={'User-Agent': ua})
-	if settings.status_code != 200:
-		return None
-	return settings.json()['data']['defaultStreamingServer']
-
-def login():
-	login = requests.get('https://www.vvvvid.it/user/login', headers={'User-Agent': ua})
-	return login.json()['data']['conn_id']
-
-def get_info(show_id, conn_id):
-	info = requests.get('https://www.vvvvid.it/vvvvid/ondemand/' + str(show_id) + '/info/?conn_id=' + conn_id, headers={'User-Agent': ua})
-	info.encoding = 'utf-8'
-	if info.json()['result'] == 'ok':
-			return info.json()['data']
-	else:
-		return False
-
-def get_seasons(show_id, conn_id):
-	seasons = requests.get('https://www.vvvvid.it/vvvvid/ondemand/' + str(show_id) + '/seasons/?conn_id=' + conn_id, headers={'User-Agent': ua})
-	if seasons.json()['result'] == 'ok' and seasons.json()['data'] and seasons.json()['data'][0]['episodes']:
-		return seasons.json()['data']
-	else:
-		return False
-
-def get_episodes(season_id, show_id, conn_id):
-	episodes = requests.get('https://www.vvvvid.it/vvvvid/ondemand/' + str(show_id) + '/season/' +str(season_id) + '?conn_id=' + conn_id, headers={'User-Agent': ua})
-	if episodes.json()['result'] == 'ok' and episodes.json()['data'] and episodes.json()['data'][0]['embed_info']:
-		return episodes.json()['data']
-	else:
-		return False
-
-vvvvid_stream_url = get_settings()
-if vvvvid_stream_url is None:
 	print("VVVVID is not available at the moment")
 	exit(1)
 
-vvvvidb = "vvvvidb.sqlite3"
+vvvvidb = VVVVIDatabase("vvvvidb.sqlite3")
-last = 0
+api.login()
 
-if not os.path.isfile(vvvvidb): 
+if not vvvvidb.is_valid:
-	con = sqlite3.connect(vvvvidb)
+	# Database file not present
-	cur = con.cursor()
+	# Create a new database
-	cur.execute("CREATE TABLE series (id INTEGER, name TEXT NOT NULL, season_id INTEGER, type TEXT, PRIMARY KEY (id, season_id));")
+	vvvvidb.create()
-	cur.execute("CREATE TABLE episodes (serie_id INTEGER, season_id INTEGER, cdn_url TEXT NOT NULL, type TEXT NOT NULL);")
-	con.commit()
-	con.close()
 else:
-	con = sqlite3.connect(vvvvidb)
+	# Database file is already present
-	cur = con.cursor()
+	# Since we have no information about series lenght prior to their pubblication
-	cur.execute("SELECT id FROM series ORDER BY id DESC LIMIT 1;")
+	# We scan all the older id to see if there are new episodes, then we scan every id greater then the last one
-	rows = cur.fetchall()
+	ids = vvvvidb.series_id()
-	if len(rows) > 0:
-		last = rows[0][0] + 1
-	con.commit() 
-	con.close()
 
-print("Resuming from...{}".format(last))
+	"""
+	for i in ids.keys():
+		print("Fetching old...{}".format(i))
+		print(ids.get(i))
+		info = api.get_info(i)
+		if not info:
+			continue
+		seasons = api.get_seasons(i)
+		for j in seasons:
+			serie = (info['show_id'], info['title'], j['season_id'], j['name'])
 
-con = sqlite3.connect(vvvvidb)
+			if j['season_id'] not in ids.get(i):
-cur = con.cursor()
+				# We found a new season for an old show
+				print("Found: {}".format(info['title']))
 
-stream_url = get_settings()
+				if not vvvvidb.insert_serie(serie):
-conn_id = login()
+					continue
 
+				episodes = get_episodes(j['season_id'], i, conn_id)
+				if not episodes:
+					continue
+				format_episodes(info, j, episodes, vvvvid_stream_url)
+				print("Found {} episodes".format(len(eps)))
+
+				vvvvidb.insert_episode(eps)
+			else:
+				# Check if there are new episodes for old season
+				episodes = get_episodes(j['season_id'], i, conn_id)
+				if not episodes:
+					continue
+				eps = format_episodes(info, j, episodes, vvvvid_stream_url)
+				print("Found {} episodes".format(len(eps)))
+
+				vvvvidb.insert_episodes(eps)
+
+		last = i
+
+	print("Resuming from...{}".format(last))
+	exit()
+	"""
+
+last = 0
+# Scan all the episodes
 for i in range(last, min(last + 500, 1000)):
 	print("Fetching...{}".format(i))
-	info = get_info(i, conn_id)
+	info = api.get_info(i)
-	if info:
+	if not info:
-		seasons = get_seasons(i, conn_id)
+		continue
-		if seasons:
+	seasons = api.get_seasons(i)
 	for j in seasons:
 		serie = (info['show_id'], info['title'], j['season_id'], j['name'])
 		print("Found: {}".format(info['title']))
 
-				try:
+		if not vvvvidb.insert_serie(serie):
-					cur.execute("INSERT INTO series (id, name, season_id, type) VALUES (?, ?, ?, ?);", serie)
+			print("Serie already present")
-					con.commit()
-				except sqlite3.IntegrityError:
-					# serie/stagione gia' presente, salta il fetch degli episodi
 			continue
 
-				eps = []
+		episodes = api.get_episodes(j['season_id'], i)
-				episodes = get_episodes(j['season_id'], i, conn_id)
+		print(len(episodes))
-				if episodes:
+		eps = api.format_episodes(j['season_id'], info['show_id'], episodes)
-					for k in episodes:
-						if k['embed_info']:
-							if k['video_type'] == 'video/rcs':
-								embed_info = ds(k['embed_info'])
-								embed_info = 'https' + embed_info[4:30] + 'i' + embed_info[31:-12] + 'master.m3u8'
-							elif k['video_type'] == 'video/vvvvid':
-								embed_info = 'https' + vvvvid_stream_url[4:] + ds(k['embed_info']) + '/playlist.m3u8'
-							elif k['video_type'] == 'video/youtube':
-								embed_info = ds(k['embed_info'])
-							elif k['video_type'] == 'video/kenc':
-								embed_info = ds(k['embed_info'])
-							else:
-								embed_info = ds(k['embed_info'])
-						eps.append((info['show_id'], j['season_id'], k['video_type'], embed_info))
 		print("Found {} episodes".format(len(eps)))
-
+		vvvvidb.insert_episodes(eps)
-					try:
-						cur.executemany("INSERT INTO episodes (serie_id, season_id, type, cdn_url) VALUES (?, ?, ?, ?);", eps)
-						con.commit()
-					except sqlite3.IntegrityError:
-						# episodi gia' presenti
-						pass
-
-con.close()