Browse Source

use sqlite database to store series

l 5 years ago
parent
commit
8b7ee3563a
3 changed files with 158 additions and 79 deletions
  1. 2 0
      .gitignore
  2. 48 0
      db_to_html.py
  3. 108 79
      vvvvget.py

+ 2 - 0
.gitignore

@@ -0,0 +1,2 @@
+index.html
+vvvvidb.sqlite3

+ 48 - 0
db_to_html.py

@@ -0,0 +1,48 @@
+import os
+import sqlite3
+
+vvvvidb = "vvvvidb.sqlite3"
+
+if not os.path.isfile(vvvvidb): 
+    print("{} file not found. Scrape the DB first using vvvvget.py".format(vvvvidb))
+    exit(1)
+
+con = sqlite3.connect(vvvvidb)
+cur = con.cursor()
+out = open('index.html', 'w')
+
+out.write('''<!DOCTYPE html>
+<html>
+<head>
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1">
+    <title>Anime | lsd.cat</title>
+    <link rel="icon" sizes="16x16" type="image/png" href="/icon.png" />
+    <style type="text/css">body{margin:40px auto;max-width:740px;line-height:1.6;color:#444;padding:0 10px}h1,h2,h3{line-height:1.2;font-size:18px;}li{font-size:12px;}</style>
+</head>
+<body>
+<a href="{}">Source Database</a><br/><br/>'''.format(vvvvidb))
+
+# select all the series
+# cur.execute("SELECT * FROM series;")
+
+# selecy only series with at least one episodes
+cur.execute("SELECT s.* FROM series as s INNER JOIN episodes as e ON e.serie_id = s.id AND e.season_id = s.season_id GROUP BY s.id, s.season_id;")
+series = cur.fetchall()
+
+for s in series:
+    serie = (str(s[0]), str(s[2]))
+    out.write("\t<h3>{}: {} - Id Serie: {} - Tipo: {}</h3>\n\t<ul>\n".format(s[0], s[1], s[2], s[3]))
+
+    cur.execute("SELECT e.cdn_url FROM episodes AS e JOIN series AS s ON e.serie_id = s.id AND e.season_id = s.season_id WHERE s.id = ? AND s.season_id = ?;", serie)
+    episodes = cur.fetchall()
+
+    [out.write("\t\t<li><a href=\"player.html#{0}\">{0}</a></li>\n".format(e[0])) for e in episodes]
+
+    out.write('\t</ul>\n')
+
+out.write('\n</body>\n</html>')
+out.close()
+
+con.commit() 
+con.close()

+ 108 - 79
vvvvget.py

@@ -1,61 +1,65 @@
+import os
 import json
 import requests
+import sqlite3
 
 def ds(h):
-    g = "MNOPIJKL89+/4567UVWXQRSTEFGHABCDcdefYZabstuvopqr0123wxyzklmnghij"
-
-    def f(m):
-        l = []
-        o = 0
-        b = False
-        m_len = len(m)
-        while ((not b) and o < m_len):
-            n = m[o] << 2
-            o += 1
-            k = -1
-            j = -1
-            if o < m_len:
-                n += m[o] >> 4
-                o += 1
-                if o < m_len:
-                    k = (m[o - 1] << 4) & 255
-                    k += m[o] >> 2
-                    o += 1
-                    if o < m_len:
-                        j = (m[o - 1] << 6) & 255
-                        j += m[o]
-                        o += 1
-                    else:
-                        b = True
-                else:
-                    b = True
-            else:
-                b = True
-            l.append(n)
-            if k != -1:
-                l.append(k)
-            if j != -1:
-                l.append(j)
-        return l
-
-    c = []
-    for e in h:
-        c.append(g.index(e))
-
-    c_len = len(c)
-    for e in range(c_len * 2 - 1, -1, -1):
-        a = c[e % c_len] ^ c[(e + 1) % c_len]
-        c[e % c_len] = a
-
-    c = f(c)
-    d = ''
-    for e in c:
-        d += chr(e)
-
-    return d
+	g = "MNOPIJKL89+/4567UVWXQRSTEFGHABCDcdefYZabstuvopqr0123wxyzklmnghij"
+
+	def f(m):
+		l = []
+		o = 0
+		b = False
+		m_len = len(m)
+		while ((not b) and o < m_len):
+			n = m[o] << 2
+			o += 1
+			k = -1
+			j = -1
+			if o < m_len:
+				n += m[o] >> 4
+				o += 1
+				if o < m_len:
+					k = (m[o - 1] << 4) & 255
+					k += m[o] >> 2
+					o += 1
+					if o < m_len:
+						j = (m[o - 1] << 6) & 255
+						j += m[o]
+						o += 1
+					else:
+						b = True
+				else:
+					b = True
+			else:
+				b = True
+			l.append(n)
+			if k != -1:
+				l.append(k)
+			if j != -1:
+				l.append(j)
+		return l
+
+	c = []
+	for e in h:
+		c.append(g.index(e))
+
+	c_len = len(c)
+	for e in range(c_len * 2 - 1, -1, -1):
+		a = c[e % c_len] ^ c[(e + 1) % c_len]
+		c[e % c_len] = a
+
+	c = f(c)
+	d = ''
+	for e in c:
+		d += chr(e)
+
+	return d
 
 def get_settings():
 	settings = requests.get('https://www.vvvvid.it/vvvvid/settings')
+	if settings.status_code != 200:
+		return None
 	return settings.json()['data']['defaultStreamingServer']
 
 def login():
@@ -85,32 +89,58 @@ def get_episodes(season_id, show_id, conn_id):
 		return False
 
 vvvvid_stream_url = get_settings()
-
-with open('index.html', 'a') as out:
-	out.write('''<!DOCTYPE html>
-<html>
-<head>
-	<meta charset="utf-8">
-	<meta name="viewport" content="width=device-width, initial-scale=1">
-	<title>Anime | lsd.cat</title>
-	<link rel="icon" sizes="16x16" type="image/png" href="/icon.png" />
-	<style type="text/css">body{margin:40px auto;max-width:740px;line-height:1.6;color:#444;padding:0 10px}h1,h2,h3{line-height:1.2;font-size:18px;}li{font-size:12px;}</style>
-</head>
-<body>''')
+if vvvvid_stream_url is None:
+	print("VVVVID is not available at the moment")
+	exit(1)
+
+vvvvidb = "vvvvidb.sqlite3"
+last = 0
+
+if not os.path.isfile(vvvvidb): 
+	con = sqlite3.connect(vvvvidb)
+	cur = con.cursor()
+	cur.execute("CREATE TABLE series (id INTEGER, name TEXT NOT NULL, season_id INTEGER, type TEXT, PRIMARY KEY (id, season_id));")
+	cur.execute("CREATE TABLE episodes (serie_id INTEGER, season_id INTEGER, cdn_url TEXT NOT NULL, type TEXT NOT NULL);")
+	con.commit()
+	con.close()
+#else:
+#	con = sqlite3.connect(vvvvidb)
+#	cur = con.cursor()
+#	cur.execute("SELECT id FROM series ORDER BY id DESC LIMIT 1;")
+#	rows = cur.fetchall()
+#	if len(rows) > 0:
+#		last = rows[0][0] + 1
+#	con.commit() 
+#	con.close()
+#
+#print("Resuming from...{}".format(last))
+
+con = sqlite3.connect(vvvvidb)
+cur = con.cursor()
 
 stream_url = get_settings()
 conn_id = login()
-for i in range(0,1000):
+
+for i in range(last, min(last + 500, 1000)):
+	print("Fetching...{}".format(i))
 	info = get_info(i, conn_id)
 	if info:
 		seasons = get_seasons(i, conn_id)
 		if seasons:
 			for j in seasons:
+				serie = (info['show_id'], info['title'], j['season_id'], j['name'])
+				print("Found: {}".format(info['title']))
+
+				try:
+					cur.execute("INSERT INTO series (id, name, season_id, type) VALUES (?, ?, ?, ?);", serie)
+					con.commit()
+				except sqlite3.IntegrityError:
+					# serie/stagione gia' presente, salta il fetch degli episodi
+					continue
+
+				eps = []
 				episodes = get_episodes(j['season_id'], i, conn_id)
 				if episodes:
-					header = '\t<h3>' + str(info['show_id']) + ': ' + info['title'] + ' Tipo: ' + j['name'] + ' Id: ' + str(j['season_id'])+'</h3>\n\t<ul>\n'
-					with open('index.html', 'a') as out:
-    						out.write(header.encode('utf-8'))
 					for k in episodes:
 						if k['embed_info']:
 							if k['video_type'] == 'video/rcs':
@@ -121,18 +151,17 @@ for i in range(0,1000):
 							elif k['video_type'] == 'video/youtube':
 								embed_info = ds(k['embed_info'])
 							elif k['video_type'] == 'video/kenc':
-								#requires flash, skipping
-								embed_info = '[FLASH ONLY] ' + ds(k['embed_info'])
+								embed_info = ds(k['embed_info'])
 							else:
-								print(k['video_type'])
-								print(ds(k['embed_info']))
-								print('Unknown video source!')
-								exit()
-						with open('index.html', 'a') as out:
-							out.write('\t\t<li><a href="player.html#' + embed_info + '">' + embed_info + '</a></li>\n')
-					with open('index.html', 'a') as out:
-						out.write('\t</ul>\n')
-with open('index.html', 'a') as out:
-	out.write('\n</body>\n</html>')
+								embed_info = ds(k['embed_info'])
+						eps.append((info['show_id'], j['season_id'], k['video_type'], embed_info))
+					print("Found {} episodes".format(len(eps)))
 
+					try:
+						cur.executemany("INSERT INTO episodes (serie_id, season_id, type, cdn_url) VALUES (?, ?, ?, ?);", eps)
+						con.commit()
+					except sqlite3.IntegrityError:
+						# episodi gia' presenti
+						pass
 
+con.close()