g/vvvvget
1
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

use sqlite database to store series

Browse Source
This commit is contained in:
l 2018-05-21 17:22:14 +02:00
parent bc043b9173
commit 8b7ee3563a
3 changed files with 152 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
index.html
vvvvidb.sqlite3

48
db_to_html.py Normal file
View File

@ -0,0 +1,48 @@
import os
import sqlite3
vvvvidb = "vvvvidb.sqlite3"
if not os.path.isfile(vvvvidb):
print("{} file not found. Scrape the DB first using vvvvget.py".format(vvvvidb))
exit(1)
con = sqlite3.connect(vvvvidb)
cur = con.cursor()
out = open('index.html', 'w')
out.write('''<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Anime | lsd.cat</title>
<link rel="icon" sizes="16x16" type="image/png" href="/icon.png" />
<style type="text/css">body{margin:40px auto;max-width:740px;line-height:1.6;color:#444;padding:0 10px}h1,h2,h3{line-height:1.2;font-size:18px;}li{font-size:12px;}</style>
</head>
<body>
<a href="{}">Source Database</a><br/><br/>'''.format(vvvvidb))
# select all the series
# cur.execute("SELECT * FROM series;")
# selecy only series with at least one episodes
cur.execute("SELECT s.* FROM series as s INNER JOIN episodes as e ON e.serie_id = s.id AND e.season_id = s.season_id GROUP BY s.id, s.season_id;")
series = cur.fetchall()
for s in series:
serie = (str(s[0]), str(s[2]))
out.write("\t<h3>{}: {} - Id Serie: {} - Tipo: {}</h3>\n\t<ul>\n".format(s[0], s[1], s[2], s[3]))
cur.execute("SELECT e.cdn_url FROM episodes AS e JOIN series AS s ON e.serie_id = s.id AND e.season_id = s.season_id WHERE s.id = ? AND s.season_id = ?;", serie)
episodes = cur.fetchall()
[out.write("\t\t<li><a href=\"player.html#{0}\">{0}</a></li>\n".format(e[0])) for e in episodes]
out.write('\t</ul>\n')
out.write('\n</body>\n</html>')
out.close()
con.commit()
con.close()

175
vvvvget.py
View File

@ -1,61 +1,65 @@
import os
import json import json
import requests import requests
import sqlite3
def ds(h): def ds(h):
g = "MNOPIJKL89+/4567UVWXQRSTEFGHABCDcdefYZabstuvopqr0123wxyzklmnghij" g = "MNOPIJKL89+/4567UVWXQRSTEFGHABCDcdefYZabstuvopqr0123wxyzklmnghij"
def f(m): def f(m):
l = [] l = []
o = 0 o = 0
b = False b = False
m_len = len(m) m_len = len(m)
while ((not b) and o < m_len): while ((not b) and o < m_len):
n = m[o] << 2 n = m[o] << 2
o += 1 o += 1
k = -1 k = -1
j = -1 j = -1
if o < m_len: if o < m_len:
n += m[o] >> 4 n += m[o] >> 4
o += 1 o += 1
if o < m_len: if o < m_len:
k = (m[o - 1] << 4) & 255 k = (m[o - 1] << 4) & 255
k += m[o] >> 2 k += m[o] >> 2
o += 1 o += 1
if o < m_len: if o < m_len:
j = (m[o - 1] << 6) & 255 j = (m[o - 1] << 6) & 255
j += m[o] j += m[o]
o += 1 o += 1
else: else:
b = True b = True
else: else:
b = True b = True
else: else:
b = True b = True
l.append(n) l.append(n)
if k != -1: if k != -1:
l.append(k) l.append(k)
if j != -1: if j != -1:
l.append(j) l.append(j)
return l return l
c = [] c = []
for e in h: for e in h:
c.append(g.index(e)) c.append(g.index(e))
c_len = len(c) c_len = len(c)
for e in range(c_len * 2 - 1, -1, -1): for e in range(c_len * 2 - 1, -1, -1):
a = c[e % c_len] ^ c[(e + 1) % c_len] a = c[e % c_len] ^ c[(e + 1) % c_len]
c[e % c_len] = a c[e % c_len] = a
c = f(c) c = f(c)
d = '' d = ''
for e in c: for e in c:
d += chr(e) d += chr(e)
return d return d
def get_settings(): def get_settings():
settings = requests.get('https://www.vvvvid.it/vvvvid/settings') settings = requests.get('https://www.vvvvid.it/vvvvid/settings')
if settings.status_code != 200:
return None
return settings.json()['data']['defaultStreamingServer'] return settings.json()['data']['defaultStreamingServer']
def login(): def login():
@ -85,32 +89,58 @@ def get_episodes(season_id, show_id, conn_id):
return False return False
vvvvid_stream_url = get_settings() vvvvid_stream_url = get_settings()
if vvvvid_stream_url is None:
print("VVVVID is not available at the moment")
exit(1)
with open('index.html', 'a') as out: vvvvidb = "vvvvidb.sqlite3"
out.write('''<!DOCTYPE html> last = 0
<html>
<head> if not os.path.isfile(vvvvidb):
<meta charset="utf-8"> con = sqlite3.connect(vvvvidb)
<meta name="viewport" content="width=device-width, initial-scale=1"> cur = con.cursor()
<title>Anime | lsd.cat</title> cur.execute("CREATE TABLE series (id INTEGER, name TEXT NOT NULL, season_id INTEGER, type TEXT, PRIMARY KEY (id, season_id));")
<link rel="icon" sizes="16x16" type="image/png" href="/icon.png" /> cur.execute("CREATE TABLE episodes (serie_id INTEGER, season_id INTEGER, cdn_url TEXT NOT NULL, type TEXT NOT NULL);")
<style type="text/css">body{margin:40px auto;max-width:740px;line-height:1.6;color:#444;padding:0 10px}h1,h2,h3{line-height:1.2;font-size:18px;}li{font-size:12px;}</style> con.commit()
</head> con.close()
<body>''') #else:
# con = sqlite3.connect(vvvvidb)
# cur = con.cursor()
# cur.execute("SELECT id FROM series ORDER BY id DESC LIMIT 1;")
# rows = cur.fetchall()
# if len(rows) > 0:
# last = rows[0][0] + 1
# con.commit()
# con.close()
#
#print("Resuming from...{}".format(last))
con = sqlite3.connect(vvvvidb)
cur = con.cursor()
stream_url = get_settings() stream_url = get_settings()
conn_id = login() conn_id = login()
for i in range(0,1000):
for i in range(last, min(last + 500, 1000)):
print("Fetching...{}".format(i))
info = get_info(i, conn_id) info = get_info(i, conn_id)
if info: if info:
seasons = get_seasons(i, conn_id) seasons = get_seasons(i, conn_id)
if seasons: if seasons:
for j in seasons: for j in seasons:
serie = (info['show_id'], info['title'], j['season_id'], j['name'])
print("Found: {}".format(info['title']))
try:
cur.execute("INSERT INTO series (id, name, season_id, type) VALUES (?, ?, ?, ?);", serie)
con.commit()
except sqlite3.IntegrityError:
# serie/stagione gia' presente, salta il fetch degli episodi
continue
eps = []
episodes = get_episodes(j['season_id'], i, conn_id) episodes = get_episodes(j['season_id'], i, conn_id)
if episodes: if episodes:
header = '\t<h3>' + str(info['show_id']) + ': ' + info['title'] + ' Tipo: ' + j['name'] + ' Id: ' + str(j['season_id'])+'</h3>\n\t<ul>\n'
with open('index.html', 'a') as out:
out.write(header.encode('utf-8'))
for k in episodes: for k in episodes:
if k['embed_info']: if k['embed_info']:
if k['video_type'] == 'video/rcs': if k['video_type'] == 'video/rcs':
@ -121,18 +151,17 @@ for i in range(0,1000):
elif k['video_type'] == 'video/youtube': elif k['video_type'] == 'video/youtube':
embed_info = ds(k['embed_info']) embed_info = ds(k['embed_info'])
elif k['video_type'] == 'video/kenc': elif k['video_type'] == 'video/kenc':
#requires flash, skipping embed_info = ds(k['embed_info'])
embed_info = '[FLASH ONLY] ' + ds(k['embed_info'])
else: else:
print(k['video_type']) embed_info = ds(k['embed_info'])
print(ds(k['embed_info'])) eps.append((info['show_id'], j['season_id'], k['video_type'], embed_info))
print('Unknown video source!') print("Found {} episodes".format(len(eps)))
exit()
with open('index.html', 'a') as out:
out.write('\t\t<li><a href="player.html#' + embed_info + '">' + embed_info + '</a></li>\n')
with open('index.html', 'a') as out:
out.write('\t</ul>\n')
with open('index.html', 'a') as out:
out.write('\n</body>\n</html>')
try:
cur.executemany("INSERT INTO episodes (serie_id, season_id, type, cdn_url) VALUES (?, ?, ?, ?);", eps)
con.commit()
except sqlite3.IntegrityError:
# episodi gia' presenti
pass
con.close()