From 5803ee46d852ff67b3af9ab3e899a16aa8b2eb50 Mon Sep 17 00:00:00 2001 From: thezero Date: Mon, 15 Oct 2018 20:27:51 +0200 Subject: [PATCH 1/2] refactor api --- .gitignore | 4 +- api.py | 113 ++++++++++++++++++++++++++++++++ db.py | 83 ++++++++++++++++++++++++ vvvvget.py | 184 +++++++++-------------------------------------------- 4 files changed, 229 insertions(+), 155 deletions(-) create mode 100644 api.py create mode 100644 db.py diff --git a/.gitignore b/.gitignore index 4fdb8d6..4340dc0 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,4 @@ index.html -vvvvidb.sqlite3 \ No newline at end of file +vvvvidb.sqlite3 + +__pycache__/ \ No newline at end of file diff --git a/api.py b/api.py new file mode 100644 index 0000000..ab7fec8 --- /dev/null +++ b/api.py @@ -0,0 +1,113 @@ +import requests + +class Api(): + ua = "Mozilla/5.0 (Windows; U; Win98; en-US; rv:0.9.4.2) Gecko/20020502 CS 2000 7.0/7.0" + + def __init__(self): + print("Staring VVVVID Api") + + def ds(self, h): + g = "MNOPIJKL89+/4567UVWXQRSTEFGHABCDcdefYZabstuvopqr0123wxyzklmnghij" + + def f(m): + l = [] + o = 0 + b = False + m_len = len(m) + while ((not b) and o < m_len): + n = m[o] << 2 + o += 1 + k = -1 + j = -1 + if o < m_len: + n += m[o] >> 4 + o += 1 + if o < m_len: + k = (m[o - 1] << 4) & 255 + k += m[o] >> 2 + o += 1 + if o < m_len: + j = (m[o - 1] << 6) & 255 + j += m[o] + o += 1 + else: + b = True + else: + b = True + else: + b = True + l.append(n) + if k != -1: + l.append(k) + if j != -1: + l.append(j) + return l + + c = [] + for e in h: + c.append(g.index(e)) + + c_len = len(c) + for e in range(c_len * 2 - 1, -1, -1): + a = c[e % c_len] ^ c[(e + 1) % c_len] + c[e % c_len] = a + + c = f(c) + d = '' + for e in c: + d += chr(e) + + return d + + def get_settings(self): + settings = requests.get('https://www.vvvvid.it/vvvvid/settings', headers={'User-Agent': Api.ua}) + if settings.status_code != 200: + return None + self.stream_url = settings.json()['data']['defaultStreamingServer'] + return self.stream_url + + def login(self): + login = requests.get('https://www.vvvvid.it/user/login', headers={'User-Agent': Api.ua}) + self.conn_id = login.json()['data']['conn_id'] + + def get_info(self, show_id): + info = requests.get('https://www.vvvvid.it/vvvvid/ondemand/' + str(show_id) + '/info/?conn_id=' + self.conn_id, headers={'User-Agent': Api.ua}) + info.encoding = 'utf-8' + if info.json()['result'] == 'ok': + return info.json()['data'] + else: + return False + + def get_seasons(self, show_id): + seasons = requests.get('https://www.vvvvid.it/vvvvid/ondemand/' + str(show_id) + '/seasons/?conn_id=' + self.conn_id, headers={'User-Agent': Api.ua}) + if seasons.json()['result'] == 'ok' and seasons.json()['data'] and seasons.json()['data'][0]['episodes']: + return seasons.json()['data'] + else: + return [] + + def get_episodes(self, season_id, show_id): + episodes = requests.get('https://www.vvvvid.it/vvvvid/ondemand/' + str(show_id) + '/season/' + str(season_id) + '?conn_id=' + self.conn_id, headers={'User-Agent': Api.ua}).json() + if episodes['result'] == 'ok' and episodes['data'] and episodes['data'][0]['embed_info']: + return episodes['data'] + else: + return [] + + def format_episodes(self, season_id, show_id, episodes): + eps = [] + count = 0 + for k in episodes: + count += 1 + if k['embed_info']: + if k['video_type'] == 'video/rcs': + embed_info = self.ds(k['embed_info']) + embed_info = 'https' + embed_info[4:30] + 'i' + embed_info[31:-12] + 'master.m3u8' + elif k['video_type'] == 'video/vvvvid': + embed_info = 'https' + self.stream_url[4:] + self.ds(k['embed_info']) + '/playlist.m3u8' + elif k['video_type'] == 'video/youtube': + embed_info = self.ds(k['embed_info']) + elif k['video_type'] == 'video/kenc': + embed_info = self.ds(k['embed_info']) + else: + embed_info = self.ds(k['embed_info']) + eps.append((count, show_id, season_id, k['video_type'], embed_info)) + return eps diff --git a/db.py b/db.py new file mode 100644 index 0000000..48c6ae7 --- /dev/null +++ b/db.py @@ -0,0 +1,83 @@ +import os +import sqlite3 + +class VVVVIDatabase(): + + def __init__(self, path): + self.path = path + self.is_valid = os.path.isfile(self.path) + + def create(self): + con = sqlite3.connect(self.path) + cur = con.cursor() + cur.execute("CREATE TABLE series (id INTEGER, name TEXT NOT NULL, season_id INTEGER, type TEXT, PRIMARY KEY (id, season_id));") + cur.execute("CREATE TABLE episodes (id INTEGER, serie_id INTEGER, season_id INTEGER, cdn_url TEXT NOT NULL, type TEXT NOT NULL);") + con.commit() + con.close() + + def last_serie(self): + con = sqlite3.connect(self.path) + cur = con.cursor() + cur.execute("SELECT id FROM series ORDER BY id DESC LIMIT 1;") + rows = cur.fetchall() + last = (rows[0][0] + 1) if len(rows) > 0 else 0 + con.commit() + con.close() + return last + + def series_id(self): + con = sqlite3.connect(self.path) + cur = con.cursor() + cur.execute("SELECT id, season_id FROM series ORDER BY id, season_id DESC;") + rows = cur.fetchall() + con.commit() + con.close() + # getting series in a useful format for later + series = {} + for i,s in rows: + if series.get(i) is None: + series[i] = [] + series[i].append(s) + return series + + def series_episodes(self, serie_id): + con = sqlite3.connect(self.path) + cur = con.cursor() + cur.execute("SELECT id, serie_id, season_id FROM episodes ORDER BY id, serie_id, season_id DESC;") + rows = cur.fetchall() + con.commit() + con.close() + # getting series in a useful format for later + series = {} + for i,s in rows: + if series.get(i) is None: + series[i] = [] + series[i].append(s) + return series + + + def insert_serie(self, serie): + con = sqlite3.connect(self.path) + cur = con.cursor() + try: + cur.execute("INSERT INTO series (id, name, season_id, type) VALUES (?, ?, ?, ?);", serie) + con.commit() + con.close() + return True + except sqlite3.IntegrityError: + # serie gia' presente + pass + return False + + def insert_episodes(self, eps): + con = sqlite3.connect(self.path) + cur = con.cursor() + try: + cur.executemany("INSERT INTO episodes (id, serie_id, season_id, type, cdn_url) VALUES (?, ?, ?, ?, ?);", eps) + con.commit() + con.close() + return True + except sqlite3.IntegrityError: + # episodi gia' presenti + pass + return False \ No newline at end of file diff --git a/vvvvget.py b/vvvvget.py index 0a1de4a..8dca700 100644 --- a/vvvvget.py +++ b/vvvvget.py @@ -2,168 +2,44 @@ import os import json import requests import sqlite3 +from db import VVVVIDatabase +from api import Api -ua = "Mozilla/5.0 (Windows; U; Win98; en-US; rv:0.9.4.2) Gecko/20020502 CS 2000 7.0/7.0" - -def ds(h): - g = "MNOPIJKL89+/4567UVWXQRSTEFGHABCDcdefYZabstuvopqr0123wxyzklmnghij" - - def f(m): - l = [] - o = 0 - b = False - m_len = len(m) - while ((not b) and o < m_len): - n = m[o] << 2 - o += 1 - k = -1 - j = -1 - if o < m_len: - n += m[o] >> 4 - o += 1 - if o < m_len: - k = (m[o - 1] << 4) & 255 - k += m[o] >> 2 - o += 1 - if o < m_len: - j = (m[o - 1] << 6) & 255 - j += m[o] - o += 1 - else: - b = True - else: - b = True - else: - b = True - l.append(n) - if k != -1: - l.append(k) - if j != -1: - l.append(j) - return l - - c = [] - for e in h: - c.append(g.index(e)) - - c_len = len(c) - for e in range(c_len * 2 - 1, -1, -1): - a = c[e % c_len] ^ c[(e + 1) % c_len] - c[e % c_len] = a - - c = f(c) - d = '' - for e in c: - d += chr(e) - - return d - -def get_settings(): - settings = requests.get('https://www.vvvvid.it/vvvvid/settings', headers={'User-Agent': ua}) - if settings.status_code != 200: - return None - return settings.json()['data']['defaultStreamingServer'] - -def login(): - login = requests.get('https://www.vvvvid.it/user/login', headers={'User-Agent': ua}) - return login.json()['data']['conn_id'] - -def get_info(show_id, conn_id): - info = requests.get('https://www.vvvvid.it/vvvvid/ondemand/' + str(show_id) + '/info/?conn_id=' + conn_id, headers={'User-Agent': ua}) - info.encoding = 'utf-8' - if info.json()['result'] == 'ok': - return info.json()['data'] - else: - return False - -def get_seasons(show_id, conn_id): - seasons = requests.get('https://www.vvvvid.it/vvvvid/ondemand/' + str(show_id) + '/seasons/?conn_id=' + conn_id, headers={'User-Agent': ua}) - if seasons.json()['result'] == 'ok' and seasons.json()['data'] and seasons.json()['data'][0]['episodes']: - return seasons.json()['data'] - else: - return False - -def get_episodes(season_id, show_id, conn_id): - episodes = requests.get('https://www.vvvvid.it/vvvvid/ondemand/' + str(show_id) + '/season/' +str(season_id) + '?conn_id=' + conn_id, headers={'User-Agent': ua}) - if episodes.json()['result'] == 'ok' and episodes.json()['data'] and episodes.json()['data'][0]['embed_info']: - return episodes.json()['data'] - else: - return False - -vvvvid_stream_url = get_settings() -if vvvvid_stream_url is None: +api = Api() +stream_url = api.get_settings() +if stream_url is None: print("VVVVID is not available at the moment") exit(1) -vvvvidb = "vvvvidb.sqlite3" -last = 0 +vvvvidb = VVVVIDatabase("vvvvidb.sqlite3") +api.login() -if not os.path.isfile(vvvvidb): - con = sqlite3.connect(vvvvidb) - cur = con.cursor() - cur.execute("CREATE TABLE series (id INTEGER, name TEXT NOT NULL, season_id INTEGER, type TEXT, PRIMARY KEY (id, season_id));") - cur.execute("CREATE TABLE episodes (serie_id INTEGER, season_id INTEGER, cdn_url TEXT NOT NULL, type TEXT NOT NULL);") - con.commit() - con.close() +if not vvvvidb.is_valid: + # Database file not present + # Create a new database + vvvvidb.create() else: - con = sqlite3.connect(vvvvidb) - cur = con.cursor() - cur.execute("SELECT id FROM series ORDER BY id DESC LIMIT 1;") - rows = cur.fetchall() - if len(rows) > 0: - last = rows[0][0] + 1 - con.commit() - con.close() - -print("Resuming from...{}".format(last)) - -con = sqlite3.connect(vvvvidb) -cur = con.cursor() - -stream_url = get_settings() -conn_id = login() + # Database file is already present + # Since we have no information about series lenght prior to their pubblication + # We scan all the older id to see if there are new episodes, then we scan every id greater then the last one + ids = vvvvidb.series_id() +last = 0 +# Scan all the episodes for i in range(last, min(last + 500, 1000)): print("Fetching...{}".format(i)) - info = get_info(i, conn_id) - if info: - seasons = get_seasons(i, conn_id) - if seasons: - for j in seasons: - serie = (info['show_id'], info['title'], j['season_id'], j['name']) - print("Found: {}".format(info['title'])) + info = api.get_info(i) + if not info: + continue + seasons = api.get_seasons(i) + for j in seasons: + print("Found: {}".format(info['title'])) - try: - cur.execute("INSERT INTO series (id, name, season_id, type) VALUES (?, ?, ?, ?);", serie) - con.commit() - except sqlite3.IntegrityError: - # serie/stagione gia' presente, salta il fetch degli episodi - continue + if not vvvvidb.insert_serie((info['show_id'], info['title'], j['season_id'], j['name'])): + print("Serie already present") + continue - eps = [] - episodes = get_episodes(j['season_id'], i, conn_id) - if episodes: - for k in episodes: - if k['embed_info']: - if k['video_type'] == 'video/rcs': - embed_info = ds(k['embed_info']) - embed_info = 'https' + embed_info[4:30] + 'i' + embed_info[31:-12] + 'master.m3u8' - elif k['video_type'] == 'video/vvvvid': - embed_info = 'https' + vvvvid_stream_url[4:] + ds(k['embed_info']) + '/playlist.m3u8' - elif k['video_type'] == 'video/youtube': - embed_info = ds(k['embed_info']) - elif k['video_type'] == 'video/kenc': - embed_info = ds(k['embed_info']) - else: - embed_info = ds(k['embed_info']) - eps.append((info['show_id'], j['season_id'], k['video_type'], embed_info)) - print("Found {} episodes".format(len(eps))) - - try: - cur.executemany("INSERT INTO episodes (serie_id, season_id, type, cdn_url) VALUES (?, ?, ?, ?);", eps) - con.commit() - except sqlite3.IntegrityError: - # episodi gia' presenti - pass - -con.close() + episodes = api.get_episodes(j['season_id'], i) + eps = api.format_episodes(j['season_id'], info['show_id'], episodes) + print("Found {} episodes".format(len(eps))) + vvvvidb.insert_episodes(eps) From 913be0e412e103bb697097d1614ffaec1040694b Mon Sep 17 00:00:00 2001 From: thezero Date: Wed, 17 Oct 2018 23:43:44 +0200 Subject: [PATCH 2/2] improve seasons/episodes fetching --- api.py | 21 ++++++++++++--------- db.py | 9 ++++----- db_to_html.py | 10 +++++----- vvvvget.py | 32 ++++++++++++++++++++++++++------ 4 files changed, 47 insertions(+), 25 deletions(-) diff --git a/api.py b/api.py index ab7fec8..8e6e4df 100644 --- a/api.py +++ b/api.py @@ -67,27 +67,28 @@ class Api(): return self.stream_url def login(self): - login = requests.get('https://www.vvvvid.it/user/login', headers={'User-Agent': Api.ua}) - self.conn_id = login.json()['data']['conn_id'] + login = requests.get('https://www.vvvvid.it/user/login', headers={'User-Agent': Api.ua}).json() + self.conn_id = login['data']['conn_id'] def get_info(self, show_id): info = requests.get('https://www.vvvvid.it/vvvvid/ondemand/' + str(show_id) + '/info/?conn_id=' + self.conn_id, headers={'User-Agent': Api.ua}) info.encoding = 'utf-8' - if info.json()['result'] == 'ok': - return info.json()['data'] + info = info.json() + if info['result'] == 'ok': + return info['data'] else: return False def get_seasons(self, show_id): - seasons = requests.get('https://www.vvvvid.it/vvvvid/ondemand/' + str(show_id) + '/seasons/?conn_id=' + self.conn_id, headers={'User-Agent': Api.ua}) - if seasons.json()['result'] == 'ok' and seasons.json()['data'] and seasons.json()['data'][0]['episodes']: - return seasons.json()['data'] + seasons = requests.get('https://www.vvvvid.it/vvvvid/ondemand/' + str(show_id) + '/seasons/?conn_id=' + self.conn_id, headers={'User-Agent': Api.ua}).json() + if seasons['result'] == 'ok' and seasons['data']: # and seasons['data'][0]['episodes']: + return seasons['data'] else: return [] def get_episodes(self, season_id, show_id): episodes = requests.get('https://www.vvvvid.it/vvvvid/ondemand/' + str(show_id) + '/season/' + str(season_id) + '?conn_id=' + self.conn_id, headers={'User-Agent': Api.ua}).json() - if episodes['result'] == 'ok' and episodes['data'] and episodes['data'][0]['embed_info']: + if episodes['result'] == 'ok' and episodes['data']: # and episodes['data'][0]['embed_info']: return episodes['data'] else: return [] @@ -97,7 +98,7 @@ class Api(): count = 0 for k in episodes: count += 1 - if k['embed_info']: + if k.get('embed_info'): if k['video_type'] == 'video/rcs': embed_info = self.ds(k['embed_info']) embed_info = 'https' + embed_info[4:30] + 'i' + embed_info[31:-12] + 'master.m3u8' @@ -109,5 +110,7 @@ class Api(): embed_info = self.ds(k['embed_info']) else: embed_info = self.ds(k['embed_info']) + else: + embed_info = "" eps.append((count, show_id, season_id, k['video_type'], embed_info)) return eps diff --git a/db.py b/db.py index 48c6ae7..31cda5e 100644 --- a/db.py +++ b/db.py @@ -34,7 +34,7 @@ class VVVVIDatabase(): con.close() # getting series in a useful format for later series = {} - for i,s in rows: + for i, s in rows: if series.get(i) is None: series[i] = [] series[i].append(s) @@ -49,12 +49,11 @@ class VVVVIDatabase(): con.close() # getting series in a useful format for later series = {} - for i,s in rows: + for i, s in rows: if series.get(i) is None: series[i] = [] series[i].append(s) - return series - + return series def insert_serie(self, serie): con = sqlite3.connect(self.path) @@ -80,4 +79,4 @@ class VVVVIDatabase(): except sqlite3.IntegrityError: # episodi gia' presenti pass - return False \ No newline at end of file + return False diff --git a/db_to_html.py b/db_to_html.py index 76c3c64..86a834b 100644 --- a/db_to_html.py +++ b/db_to_html.py @@ -3,7 +3,7 @@ import sqlite3 vvvvidb = "vvvvidb.sqlite3" -if not os.path.isfile(vvvvidb): +if not os.path.isfile(vvvvidb): print("{} file not found. Scrape the DB first using vvvvget.py".format(vvvvidb)) exit(1) @@ -23,10 +23,10 @@ out.write(''' Source Database

''') -# select all the series +# Select all the series # cur.execute("SELECT * FROM series;") -# selecy only series with at least one episodes +# Selecy only series with at least one episodes cur.execute("SELECT s.* FROM series as s INNER JOIN episodes as e ON e.serie_id = s.id AND e.season_id = s.season_id GROUP BY s.id, s.season_id;") series = cur.fetchall() @@ -34,7 +34,7 @@ for s in series: serie = (str(s[0]), str(s[2])) out.write("\t

{}: {} - Id Serie: {} - Tipo: {}

\n\t