from datetime import datetime, timezone
import bcrypt
-from flask import Blueprint, render_template, redirect, flash, g, request, current_app, session
+from flask import abort, Blueprint, render_template, redirect, flash, g, request, current_app, session
from . import comments, db
from .logutils import flash_and_log
return _wrapper
+def admin_only(f):
+ @functools.wraps(f)
+ def _wrapper(*args, **kwargs):
+ if request.remote_addr != "127.0.0.1":
+ current_app.logger.info(
+ "auth: Blocked attempt to access admin endpoint "
+ f"{request.endpoint} from {request.remote_addr}")
+ abort(404) # Pretend this endpoint doesn't exist
+ return f(*args, **kwargs)
+
+ return _wrapper
+
from . import datadir
-DB_VERSION = 7
+DB_VERSION = 8
def get():
db = getattr(g, '_database', None)
from datetime import datetime, timezone
-from flask import abort, Blueprint, get_flashed_messages, session, redirect, \
- render_template, request
+from flask import abort, Blueprint, current_app, get_flashed_messages, session, \
+ redirect, render_template, request
-from . import db
+from . import auth, db
from .logutils import flash_and_log
bp = Blueprint("dreams-importer", __name__, url_prefix="/dreams-importer")
return render_template("dreams-importer.html")
@bp.get("/next-in-queue")
+@auth.admin_only
def get_next_in_queue():
# Check for pending imports, set error status
result = db.query("SELECT * FROM import_queue WHERE status = 1")
db.query(
"UPDATE import_queue SET status = 2 WHERE queueid = ?",
[row["queueid"]])
+ current_app.logger.info(f"dreams_importer: Deleted {row['queueid']} from queue")
# Get next queued import
result = db.query(
"""
SELECT * FROM import_queue
+ INNER JOIN songs USING (songid)
WHERE status = 0
LIMIT 1
""", one=True)
"UPDATE import_queue SET status = 1 WHERE queueid = ?",
[result["queueid"]])
+ current_app.logger.info(f"dreams_importer: Starting import for {result['queueid']}")
+ else:
+ current_app.logger.info(f"dreams_importer: No more songs to import")
+
db.commit()
return response
-# TODO: Local/admin only
@bp.get("/reset-queue")
+@auth.admin_only
def reset_queue():
# Reset all pending (1) and error (2) statuses to queued (0)
result = db.query("SELECT * FROM import_queue WHERE status = 1 OR status = 2")
db.query(
"UPDATE import_queue SET status = 0 WHERE queueid = ?",
[row["queueid"]])
+ current_app.logger.info(f"dreams_importer: Reset import for {row['queueid']}")
db.commit()
return {"status": "ok"}
-# TODO: Local/admin only
@bp.get("/delete/<int:queueid>")
+@auth.admin_only
def delete_entry(queueid):
- db.query("DELETE FROM import_queue WHERE queueid = ?", [queueid])
+ delete_from_queue(queueid)
db.commit()
return {"status": "ok"}
+def delete_from_queue(queueid):
+ db.query("DELETE FROM import_queue WHERE queueid = ?", [queueid])
+ current_app.logger.info(f"dreams_importer: Removed {queueid} from queue")
+
+def add_to_queue(songid, indreams_url):
+ print(songid, indreams_url)
+ timestamp = datetime.now(timezone.utc).isoformat()
+ result = db.query(
+ """
+ INSERT INTO import_queue (created, indreamsurl, songid, status)
+ VALUES (?, ?, ?, 0)
+ RETURNING queueid
+ """,
+ [timestamp, indreams_url, songid],
+ expect_one=True)
+ queueid = result["queueid"]
+
+ db.query(
+ """
+ UPDATE songs
+ SET queueid = ?
+ WHERE songid = ?
+ """,
+ [queueid, songid])
+
+ current_app.logger.info(f"dreams_importer: Added {queueid} to queue")
+ flash_and_log(f"Queued for import from Dreams: {indreams_url}")
+
+ return queueid
+
from yt_dlp import YoutubeDL
from yt_dlp.utils import DownloadError
-from . import auth, comments, colors, datadir, db, users
+from . import auth, comments, colors, datadir, db, dreams_importer, users
from .sanitize import sanitize_user_text
from .logutils import flash_and_log
hidden: bool
eventid: Optional[int]
jamid: Optional[int]
+ queueid: Optional[int]
+ queue_status: Optional[int]
event_title: Optional[str]
def json(self):
def get_comments(self):
return comments.for_thread(self.threadid)
+ def get_queue_position(self):
+ if self.queueid is None:
+ return 0
+ preceding_items = db.query(
+ "SELECT * FROM import_queue WHERE status = 0 AND queueid < ?",
+ [self.queueid])
+ return len(preceding_items) + 1
+
def by_id(songid):
songs = _from_db("SELECT * FROM songs_view WHERE songid = ?", [songid])
if not songs:
song_tags = sd["tags"].split(",") if sd["tags"] else []
song_collabs = sd["collaborators"].split(",") if sd["collaborators"] else []
- # Song is hidden if it was submitted to an event that hasn't ended yet
- hidden = False
+ # Song is hidden if it is still queued for import or was submitted to
+ # an event that hasn't ended yet
+ hidden = sd["queueid"] != None
if sd["event_enddate"]:
enddate = datetime.fromisoformat(sd["event_enddate"])
- hidden = datetime.now(timezone.utc) < enddate
+ hidden |= datetime.now(timezone.utc) < enddate
created = (
datetime.fromisoformat(sd["created"])
hidden=hidden,
eventid=sd["eventid"],
jamid=sd["jamid"],
+ queueid=sd["queueid"],
+ queue_status=sd["queue_status"],
event_title=sd["event_title"],
))
return songs
abort(400)
file = request.files["song-file"] if "song-file" in request.files else None
- yt_url = request.form["song-url"] if "song-url" in request.form else None
+ url = request.form["song-url"] if "song-url" in request.form else None
title = request.form["title"]
description = request.form["description"]
+ upload_type = request.form["upload-type"]
tags = [t.strip() for t in request.form["tags"].split(",") if t]
collaborators = [c.strip() for c in request.form["collabs"].split(",") if c]
abort(401)
error = False
- if file or yt_url:
+ if file or url:
with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
- passed = convert_song(tmp_file, file, yt_url)
+ passed = convert_song(tmp_file, file, url, upload_type)
if passed:
# Move file to permanent location
- user_songs_path = datadir.get_user_songs_path(session["userid"])
- filepath = user_songs_path / (str(song_data["songid"]) + ".mp3")
- shutil.move(tmp_file.name, filepath)
+ if upload_type != "dreams":
+ user_songs_path = datadir.get_user_songs_path(session["userid"])
+ filepath = user_songs_path / (str(song_data["songid"]) + ".mp3")
+ shutil.move(tmp_file.name, filepath)
else:
error = True
""",
[collab, songid])
+ if upload_type == "dreams":
+ if song_data["queueid"] is not None:
+ dreams_importer.delete_from_queue(song_data["queueid"])
+ dreams_importer.add_to_queue(songid, url)
+
db.commit()
flash_and_log(f"Successfully updated '{title}'", "success")
def create_song():
file = request.files["song-file"] if "song-file" in request.files else None
- yt_url = request.form["song-url"] if "song-url" in request.form else None
+ url = request.form["song-url"] if "song-url" in request.form else None
title = request.form["title"]
description = request.form["description"]
+ upload_type = request.form["upload-type"]
tags = [t.strip() for t in request.form["tags"].split(",") if t]
collaborators = [c.strip() for c in request.form["collabs"].split(",") if c]
try:
abort(400)
with tempfile.NamedTemporaryFile(delete=False) as tmp_file:
- passed = convert_song(tmp_file, file, yt_url)
+ passed = convert_song(tmp_file, file, url, upload_type)
if not passed:
return True
one=True)
# Move file to permanent location
- user_songs_path = datadir.get_user_songs_path(session["userid"])
- filepath = user_songs_path / (str(song_data["songid"]) + ".mp3")
- shutil.move(tmp_file.name, filepath)
+ if upload_type != "dreams":
+ user_songs_path = datadir.get_user_songs_path(session["userid"])
+ filepath = user_songs_path / (str(song_data["songid"]) + ".mp3")
+ shutil.move(tmp_file.name, filepath)
# Assign tags
songid = song_data["songid"]
"INSERT INTO song_collaborators (songid, name) VALUES (?, ?)",
[songid, collab])
+ if upload_type == "dreams":
+ dreams_importer.add_to_queue(songid, url)
+
db.commit()
flash_and_log(f"Successfully uploaded '{title}'", "success")
return False # No error
-def convert_song(tmp_file, request_file, yt_url):
- if request_file:
+def convert_song(tmp_file, request_file, url, upload_type):
+ if upload_type == "file":
# Get uploaded file
request_file.save(tmp_file)
tmp_file.close()
- else:
+ elif upload_type == "yt":
# Import from YouTube
tmp_file.close()
os.unlink(tmp_file.name) # Delete file so yt-dlp doesn't complain
try:
- yt_import(tmp_file, yt_url)
+ yt_import(tmp_file, url)
except DownloadError as ex:
current_app.logger.warning(str(ex))
- flash_and_log(f"Failed to import from YouTube URL: {yt_url}")
+ flash_and_log(f"Failed to import from YouTube URL: {url}")
return False
+ elif upload_type == "dreams":
+ # Queue for dreams importer
+ return True
# Try to convert with ffmpeg
with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as out_file:
flash_and_log("Invalid audio file", "error")
return False
-def yt_import(tmp_file, yt_url):
+def yt_import(tmp_file, url):
ydl_opts = {
'format': 'm4a/bestaudio/best',
'outtmpl': tmp_file.name,
'logger': current_app.logger,
}
with YoutubeDL(ydl_opts) as ydl:
- ydl.download([yt_url])
+ ydl.download([url])
@bp.get("/delete-song/<int:songid>")
@auth.requires_login
LEFT JOIN collaborators_agg ON collaborators_agg.songid = songs.songid
LEFT JOIN jam_events ON jam_events.eventid = songs.eventid;
-DROP TABLE IF EXISTS users_push_subscriptions
+DROP TABLE IF EXISTS users_push_subscriptions;
CREATE TABLE users_push_subscriptions (
subid INTEGER PRIMARY KEY,
userid INTEGER NOT NULL,
-- DROP TABLE IF EXISTS import_queue;
CREATE TABLE import_queue (
queueid INTEGER PRIMARY KEY AUTOINCREMENT,
- indreamsurl TEXT NOT NULL,
created TEXT NOT NULL,
+ indreamsurl TEXT NOT NULL,
songid INTEGER NOT NULL REFERENCES songs(songid) ON DELETE CASCADE,
status INTEGER NOT NULL
);
+DROP VIEW IF EXISTS songs_view;
+CREATE VIEW songs_view AS
+ WITH
+ tags_agg AS (
+ SELECT songid, GROUP_CONCAT(tag) as tags
+ FROM song_tags
+ GROUP BY songid
+ ),
+ collaborators_agg AS (
+ SELECT songid, GROUP_CONCAT(name) as collaborators
+ FROM song_collaborators
+ GROUP BY songid
+ )
+ SELECT
+ songs.*,
+ users.username,
+ users.fgcolor,
+ users.bgcolor,
+ users.accolor,
+ jam_events.title AS event_title,
+ jam_events.jamid AS jamid,
+ jam_events.enddate AS event_enddate,
+ tags_agg.tags,
+ collaborators_agg.collaborators,
+ import_queue.status AS queue_status
+ FROM songs
+ INNER JOIN users USING (userid)
+ LEFT JOIN tags_agg USING (songid)
+ LEFT JOIN collaborators_agg USING (songid)
+ LEFT JOIN jam_events USING (eventid)
+ LEFT JOIN import_queue USING (queueid);
+
PRAGMA user_version = 8;
<div class="song-info">
<!-- Song Title -->
<div class="song-title">
- {%- if song.hidden %}<span class="visibility-indicator" title="This song is not visible to others until the end of the event">[Hidden]</span>{% endif %}
+ {%- if song.hidden %}<span class="visibility-indicator" title="This song is not visible to others">[Hidden]</span>{% endif %}
+ {%- if song.queue_status == 0 %}<span class="visibility-indicator" title="Position in importer queue">[Queue Pos: {{ song.get_queue_position() }}]</span>{% endif %}
+ {%- if song.queue_status == 1 %}<span class="visibility-indicator">[Importing Now]</span>{% endif %}
+ {%- if song.queue_status == 2 %}<span class="visibility-indicator">[Import Failed]</span>{% endif %}
<a href="/song/{{ song.userid }}/{{ song.songid }}?action=view">{{ song.title }}</a>
</div>
--- /dev/null
+from .utils import upload_song
+
+TEST_URL = "https://indreams.me/element/my-element"
+
+def test_import_failure(client, user):
+ upload_song(
+ client, b"Queued for import from Dreams",
+ upload_type="dreams",
+ song_url=TEST_URL)
+ response = client.get(f"/users/user")
+ assert b"[Hidden]" in response.data
+ assert b"[Queue Pos: 1]" in response.data
+
+ response = client.get("/dreams-importer/next-in-queue")
+ assert response.json["next"]["indreamsurl"] == TEST_URL
+ assert response.json["next"]["userid"] == 1
+ assert response.json["next"]["songid"] == 1
+ assert response.json["next"]["queueid"] == 1
+
+ # Now in progress
+ response = client.get(f"/users/user")
+ assert b"[Hidden]" in response.data
+ assert b"[Importing Now]" in response.data
+
+ # No next item - but signals failure
+ response = client.get("/dreams-importer/next-in-queue")
+ assert response.json["next"] is None
+
+ # Now failed
+ response = client.get(f"/users/user")
+ assert b"[Hidden]" in response.data
+ assert b"[Import Failed]" in response.data
+
+def test_import_success(client, user):
+ upload_song(
+ client, b"Queued for import from Dreams",
+ upload_type="dreams",
+ song_url=TEST_URL)
+
+ # Delete (signals success)
+ response = client.get("/dreams-importer/delete/1")
+ assert response.json["status"] == "ok"
+
+ # Now unhidden
+ response = client.get(f"/users/user")
+ assert b"[Hidden]" not in response.data
+ assert b"[Import Failed]" not in response.data
+
+def test_reset_queue_after_error(client, user):
+ upload_song(
+ client, b"Queued for import from Dreams",
+ upload_type="dreams",
+ song_url=TEST_URL)
+
+ # In queue
+ response = client.get("/dreams-importer/next-in-queue")
+ assert response.json["next"]["indreamsurl"] == TEST_URL
+
+ # No next item - but signals failure
+ response = client.get("/dreams-importer/next-in-queue")
+ assert response.json["next"] is None
+
+ # Reset
+ response = client.get("/dreams-importer/reset-queue")
+ assert response.json["status"] == "ok"
+
+ # Now queued again
+ response = client.get(f"/users/user")
+ assert b"[Hidden]" in response.data
+ assert b"[Queue Pos: 1]" in response.data
+
+ # Get next again
+ response = client.get("/dreams-importer/next-in-queue")
+ assert response.json["next"]["indreamsurl"] == TEST_URL
+
+def test_reset_queue_after_pending(client, user):
+ upload_song(
+ client, b"Queued for import from Dreams",
+ upload_type="dreams",
+ song_url=TEST_URL)
+
+ # In queue - now pending
+ response = client.get("/dreams-importer/next-in-queue")
+ assert response.json["next"]["indreamsurl"] == TEST_URL
+
+ # Reset
+ response = client.get("/dreams-importer/reset-queue")
+ assert response.json["status"] == "ok"
+
+ # Now queued again
+ response = client.get(f"/users/user")
+ assert b"[Hidden]" in response.data
+ assert b"[Queue Pos: 1]" in response.data
+
+ # Get next again
+ response = client.get("/dreams-importer/next-in-queue")
+ assert response.json["next"]["indreamsurl"] == TEST_URL
+
+def test_update_queued_song(client, user):
+ upload_song(
+ client, b"Queued for import from Dreams",
+ upload_type="dreams",
+ song_url=TEST_URL)
+
+ # Initial queue position
+ response = client.get(f"/users/user")
+ assert b"[Hidden]" in response.data
+ assert b"[Queue Pos: 1]" in response.data
+
+ # In queue - now pending
+ response = client.get("/dreams-importer/next-in-queue")
+ assert response.json["next"]["indreamsurl"] == TEST_URL
+
+ # Request a new upload
+ upload_song(
+ client, b"Queued for import from Dreams",
+ upload_type="dreams",
+ song_url=TEST_URL + "_new",
+ songid=1)
+
+ # Now queued again (replaced original)
+ response = client.get(f"/users/user")
+ assert b"[Hidden]" in response.data
+ assert b"[Queue Pos: 1]" in response.data
+
+ # In queue - now pending with new URL
+ response = client.get("/dreams-importer/next-in-queue")
+ assert response.json["next"]["indreamsurl"] == TEST_URL + "_new"
+ assert response.json["next"]["songid"] == 1
+
"description": "song description",
"tags": "tag",
"collabs": "collab",
+ "upload-type": "yt",
}
response = client.post("/upload-song", data=data)
assert response.status_code == 302
"description": "song description",
"tags": "tag",
"collabs": "collab",
+ "upload-type": "yt",
}
response = client.post("/upload-song?songid=1", data=data)
assert response.status_code == 302
"description": "song description",
"tags": "tag",
"collabs": "collab",
+ "upload-type": "file",
}
response = client.post(f"/upload-song?songid=2", data=data)
"description": "song description",
"tags": "tag",
"collabs": "collab",
+ "upload-type": "file",
}
response = client.post(f"/upload-song?songid=abc", data=data)
"description": "song description",
"tags": "tag",
"collabs": "collab",
+ "upload-type": "file",
}
response = client.post(f"/upload-song?songid=1", data=data)
def upload_song(
client, msg, error=False, songid=None, eventid=None,
- user="user", userid=1, filename=TEST_DATA/"sample-3s.mp3", **kwargs):
+ user="user", userid=1, filename=TEST_DATA/"sample-3s.mp3", upload_type="file", song_url=None, **kwargs):
- song_file = open(filename, "rb")
+ song_file = None
+ if filename:
+ song_file = open(filename, "rb")
data = {
"song-file": song_file,
"description": "song description",
"tags": "tag",
"collabs": "collab",
+ "upload-type": upload_type,
+ "song-url": song_url,
}
for k, v in kwargs.items():
data[k] = v
projects / littlesongplace.git / commitdiff