g/vvvvget
1
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

use sqlite database to store series

Browse Source
This commit is contained in:
l 2018-05-21 17:22:14 +02:00
parent bc043b9173
commit 8b7ee3563a
3 changed files with 152 additions and 73 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
index.html
vvvvidb.sqlite3

48
db_to_html.py Normal file
View File

@ -0,0 +1,48 @@
import os
import sqlite3
vvvvidb = "vvvvidb.sqlite3"
if not os.path.isfile(vvvvidb):
print("{} file not found. Scrape the DB first using vvvvget.py".format(vvvvidb))
exit(1)
con = sqlite3.connect(vvvvidb)
cur = con.cursor()
out = open('index.html', 'w')
out.write('''<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Anime | lsd.cat</title>
<link rel="icon" sizes="16x16" type="image/png" href="/icon.png" />
<style type="text/css">body{margin:40px auto;max-width:740px;line-height:1.6;color:#444;padding:0 10px}h1,h2,h3{line-height:1.2;font-size:18px;}li{font-size:12px;}</style>
</head>
<body>
<a href="{}">Source Database</a><br/><br/>'''.format(vvvvidb))
# select all the series
# cur.execute("SELECT * FROM series;")
# selecy only series with at least one episodes
cur.execute("SELECT s.* FROM series as s INNER JOIN episodes as e ON e.serie_id = s.id AND e.season_id = s.season_id GROUP BY s.id, s.season_id;")
series = cur.fetchall()
for s in series:
serie = (str(s[0]), str(s[2]))
out.write("\t<h3>{}: {} - Id Serie: {} - Tipo: {}</h3>\n\t<ul>\n".format(s[0], s[1], s[2], s[3]))
cur.execute("SELECT e.cdn_url FROM episodes AS e JOIN series AS s ON e.serie_id = s.id AND e.season_id = s.season_id WHERE s.id = ? AND s.season_id = ?;", serie)
episodes = cur.fetchall()
[out.write("\t\t<li><a href=\"player.html#{0}\">{0}</a></li>\n".format(e[0])) for e in episodes]
out.write('\t</ul>\n')
out.write('\n</body>\n</html>')
out.close()
con.commit()
con.close()

83
vvvvget.py
View File

@ -1,5 +1,7 @@
import os
import json
import requests
import sqlite3
def ds(h):
g = "MNOPIJKL89+/4567UVWXQRSTEFGHABCDcdefYZabstuvopqr0123wxyzklmnghij"
@ -56,6 +58,8 @@ def ds(h):
def get_settings():
settings = requests.get('https://www.vvvvid.it/vvvvid/settings')
if settings.status_code != 200:
return None
return settings.json()['data']['defaultStreamingServer']
def login():
@ -85,32 +89,58 @@ def get_episodes(season_id, show_id, conn_id):
return False
vvvvid_stream_url = get_settings()
if vvvvid_stream_url is None:
print("VVVVID is not available at the moment")
exit(1)
with open('index.html', 'a') as out:
out.write('''<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Anime | lsd.cat</title>
<link rel="icon" sizes="16x16" type="image/png" href="/icon.png" />
<style type="text/css">body{margin:40px auto;max-width:740px;line-height:1.6;color:#444;padding:0 10px}h1,h2,h3{line-height:1.2;font-size:18px;}li{font-size:12px;}</style>
</head>
<body>''')
vvvvidb = "vvvvidb.sqlite3"
last = 0
if not os.path.isfile(vvvvidb):
con = sqlite3.connect(vvvvidb)
cur = con.cursor()
cur.execute("CREATE TABLE series (id INTEGER, name TEXT NOT NULL, season_id INTEGER, type TEXT, PRIMARY KEY (id, season_id));")
cur.execute("CREATE TABLE episodes (serie_id INTEGER, season_id INTEGER, cdn_url TEXT NOT NULL, type TEXT NOT NULL);")
con.commit()
con.close()
#else:
# con = sqlite3.connect(vvvvidb)
# cur = con.cursor()
# cur.execute("SELECT id FROM series ORDER BY id DESC LIMIT 1;")
# rows = cur.fetchall()
# if len(rows) > 0:
# last = rows[0][0] + 1
# con.commit()
# con.close()
#
#print("Resuming from...{}".format(last))
con = sqlite3.connect(vvvvidb)
cur = con.cursor()
stream_url = get_settings()
conn_id = login()
for i in range(0,1000):
for i in range(last, min(last + 500, 1000)):
print("Fetching...{}".format(i))
info = get_info(i, conn_id)
if info:
seasons = get_seasons(i, conn_id)
if seasons:
for j in seasons:
serie = (info['show_id'], info['title'], j['season_id'], j['name'])
print("Found: {}".format(info['title']))
try:
cur.execute("INSERT INTO series (id, name, season_id, type) VALUES (?, ?, ?, ?);", serie)
con.commit()
except sqlite3.IntegrityError:
# serie/stagione gia' presente, salta il fetch degli episodi
continue
eps = []
episodes = get_episodes(j['season_id'], i, conn_id)
if episodes:
header = '\t<h3>' + str(info['show_id']) + ': ' + info['title'] + ' Tipo: ' + j['name'] + ' Id: ' + str(j['season_id'])+'</h3>\n\t<ul>\n'
with open('index.html', 'a') as out:
out.write(header.encode('utf-8'))
for k in episodes:
if k['embed_info']:
if k['video_type'] == 'video/rcs':
@ -121,18 +151,17 @@ for i in range(0,1000):
elif k['video_type'] == 'video/youtube':
embed_info = ds(k['embed_info'])
elif k['video_type'] == 'video/kenc':
#requires flash, skipping
embed_info = '[FLASH ONLY] ' + ds(k['embed_info'])
embed_info = ds(k['embed_info'])
else:
print(k['video_type'])
print(ds(k['embed_info']))
print('Unknown video source!')
exit()
with open('index.html', 'a') as out:
out.write('\t\t<li><a href="player.html#' + embed_info + '">' + embed_info + '</a></li>\n')
with open('index.html', 'a') as out:
out.write('\t</ul>\n')
with open('index.html', 'a') as out:
out.write('\n</body>\n</html>')
embed_info = ds(k['embed_info'])
eps.append((info['show_id'], j['season_id'], k['video_type'], embed_info))
print("Found {} episodes".format(len(eps)))
try:
cur.executemany("INSERT INTO episodes (serie_id, season_id, type, cdn_url) VALUES (?, ?, ?, ?);", eps)
con.commit()
except sqlite3.IntegrityError:
# episodi gia' presenti
pass
con.close()