Add last.fm backend importer - interim check in

Dockerfile

@@ -153,6 +153,12 @@ COPY ./docker/services/spotify_reader/spotify_reader.service /etc/service/spotif
 COPY ./docker/services/spotify_reader/spotify_reader.finish /etc/service/spotify_reader/finish
 RUN touch /etc/service/spotify_reader/down
 
+# Last.fm importer
+COPY ./docker/services/lastfm_importer/consul-template-lastfm-importer.conf /etc/consul-template-lastfm-importer.conf
+COPY ./docker/services/lastfm_importer/lastfm_importer.service /etc/service/lastfm_importer/run
+COPY ./docker/services/lastfm_importer/lastfm_importer.finish /etc/service/lastfm_importer/finish
+RUN touch /etc/service/lastfm_importer/down
+
 # Timescale writer
 COPY ./docker/services/timescale_writer/consul-template-timescale-writer.conf /etc/consul-template-timescale-writer.conf
 COPY ./docker/services/timescale_writer/timescale_writer.service /etc/service/timescale_writer/run

admin/sql/create_tables.sql

@@ -126,7 +126,7 @@ CREATE TABLE external_service_oauth (
     user_id                 INTEGER NOT NULL,  -- FK to "user".id
     external_user_id        TEXT,
     service                 external_service_oauth_type NOT NULL,
-    access_token            TEXT NOT NULL,
+    access_token            TEXT,
     refresh_token           TEXT,
     token_expires           TIMESTAMP WITH TIME ZONE,
     last_updated            TIMESTAMP WITH TIME ZONE DEFAULT NOW(),

admin/sql/updates/2024-08-02-make-external-service-access-token-nullable.sql

@@ -0,0 +1,5 @@
+BEGIN;
+
+ALTER TABLE external_service_oauth ALTER COLUMN access_token DROP NOT NULL;
+
+COMMIT;

docker/docker-compose.yml

@@ -106,13 +106,24 @@ services:
     image: web
     volumes:
       - ..:/code/listenbrainz:z
-    command: python3 -m "listenbrainz.spotify_updater.spotify_read_listens"
+    command: python3 -m "listenbrainz.listens_importer.spotify"
     user: "${LB_DOCKER_USER:-root}:${LB_DOCKER_GROUP:-root}"
     depends_on:
       - redis
       - lb_db
       - rabbitmq
 
+  lastfm_importer:
+    image: web
+    volumes:
+      - ..:/code/listenbrainz:z
+    command: python3 -m "listenbrainz.listens_importer.lastfm"
+    user: "${LB_DOCKER_USER:-root}:${LB_DOCKER_GROUP:-root}"
+    depends_on:
+      - redis
+      - rabbitmq
+      - lb_db
+
   websockets:
     image: web
     volumes:

docker/rc.local

@@ -43,6 +43,13 @@ then
     rm -f /etc/service/spotify_reader/down
 fi
 
+if [ "${CONTAINER_NAME}" = "listenbrainz-lastfm-reader-${DEPLOY_ENV}" ]
+then
+    log Enabling last.fm importer
+    rm -f /etc/service/lastfm_importer/down
+fi
+
+
 if [ "${CONTAINER_NAME}" = "listenbrainz-spark-reader-${DEPLOY_ENV}" ]
 then
     log Enabling spark reader

docker/services/lastfm_importer/consul-template-lastfm-importer.conf

@@ -0,0 +1,12 @@
+template {
+    source = "/code/listenbrainz/consul_config.py.ctmpl"
+    destination = "/code/listenbrainz/listenbrainz/config.py"
+}
+
+exec {
+    command = ["run-lb-command", "python3", "-m", "listenbrainz.listens_importer.lastfm"]
+    splay = "5s"
+    reload_signal = "SIGHUP"
+    kill_signal = "SIGTERM"
+    kill_timeout = "30s"
+}

docker/services/lastfm_importer/lastfm_importer.finish

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+export service="lastfm-importer"
+
+. /etc/lb-startup-common.sh
+
+
+generate_message "$service" "$@"
+
+log "$message"
+
+send_sentry_message "$message"
+
+if [ "$1" != "0" ]; then
+  log "Exited with non-0 status, sleeping 10 seconds"
+  sleep 10
+fi

docker/services/lastfm_importer/lastfm_importer.service

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+sleep 1
+exec run-consul-template -config /etc/consul-template-lastfm-importer.conf

docker/services/spotify_reader/consul-template-spotify-reader.conf

@@ -4,7 +4,7 @@ template {
 }
 
 exec {
-    command = ["run-lb-command", "python3", "-m", "listenbrainz.spotify_updater.spotify_read_listens"]
+    command = ["run-lb-command", "python3", "-m", "listenbrainz.listens_importer.spotify"]
     splay = "5s"
     reload_signal = "SIGHUP"
     kill_signal = "SIGTERM"

frontend/css/music-services.less

@@ -1,6 +1,11 @@
 @checkbox-size: 40px;
 
 .music-service-selection {
+  button.music-service-option{
+    border: none;
+    background: none;
+    padding: 0;
+  }
   .music-service-option {
     input[type="radio"] {
       display: none;

frontend/js/src/settings/music-services/details/MusicServices.tsx

@@ -18,6 +18,7 @@ type MusicServicesLoaderData = {
   current_critiquebrainz_permissions: string;
   current_soundcloud_permissions: string;
   current_apple_permissions: string;
+  current_lastfm_permissions: string;
 };
 
 export default function MusicServices() {
@@ -34,6 +35,7 @@ export default function MusicServices() {
     critiquebrainz: loaderData.current_critiquebrainz_permissions,
     soundcloud: loaderData.current_soundcloud_permissions,
     appleMusic: loaderData.current_apple_permissions,
+    lastFm: loaderData.current_lastfm_permissions,
   });
 
   const handlePermissionChange = async (
@@ -161,6 +163,56 @@ export default function MusicServices() {
     }
   };
 
+  const handleConnectToLaftFM = async (
+    evt: React.FormEvent<HTMLFormElement>
+  ) => {
+    evt.preventDefault();
+    const formData = new FormData(evt.currentTarget);
+    const username = formData.get("lastfmUsername");
+    const startdate = formData.get("lastFMStartDatetime");
+    try {
+      const response = await fetch(`/settings/music-services/lastfm/connect/`, {
+        method: "POST",
+        body: JSON.stringify({
+          external_user_id: username,
+          latest_listened_at: startdate || null,
+        }),
+        headers: {
+          "Content-Type": "application/json",
+        },
+      });
+
+      if (response.ok) {
+        toast.success(
+          <ToastMsg
+            title="Success"
+            message="Your Last.FM account is connected to ListenBrainz"
+          />
+        );
+
+        setPermissions((prevState) => ({
+          ...prevState,
+          lastfm: "import",
+        }));
+      } else {
+        if (response.bodyUsed) {
+          const body = await response.json();
+          if (body.error) {
+            throw body.error;
+          }
+        }
+        throw response.statusText;
+      }
+    } catch (error) {
+      toast.error(
+        <ToastMsg
+          title="Failed to connect to Last.FM"
+          message={error.toString()}
+        />
+      );
+    }
+  };
+
   return (
     <>
       <Helmet>
@@ -290,6 +342,84 @@ export default function MusicServices() {
           </div>
         </div>
 
+        <div className="panel panel-default">
+          <div className="panel-heading">
+            <h3 className="panel-title">Last.FM</h3>
+          </div>
+          <div className="panel-body">
+            <p>
+              Connect to your Last.FM account to automatically add your
+              scrobbles to your ListenBrainz listens.
+            </p>
+            <p className="alert alert-warning">
+              You must first disable the &#34;Hide recent listening
+              information&#34; setting in your Last.fm{" "}
+              <a
+                href="https://www.last.fm/settings/privacy"
+                target="_blank"
+                rel="noreferrer"
+              >
+                privacy settings
+              </a>
+              .
+            </p>
+            <form onSubmit={handleConnectToLaftFM}>
+              <div className="flex flex-wrap" style={{ gap: "1em" }}>
+                <div>
+                  <label htmlFor="lastfmUsername">Your Last.FM username:</label>
+                  <input
+                    type="text"
+                    className="form-control"
+                    name="lastfmUsername"
+                    title="Last.FM Username"
+                    placeholder="Last.FM Username"
+                  />
+                </div>
+                <div>
+                  <label htmlFor="datetime">
+                    Start import from (optional):
+                  </label>
+                  <input
+                    type="datetime-local"
+                    className="form-control"
+                    max={new Date().toISOString()}
+                    name="lastFMStartDatetime"
+                    title="Date and time to start import at"
+                  />
+                </div>
+              </div>
+              <br />
+              <div className="music-service-selection">
+                <button type="submit" className="music-service-option">
+                  <input
+                    readOnly
+                    type="radio"
+                    id="lastfm_import"
+                    name="lastfm"
+                    value="import"
+                    checked={permissions.lastFm === "import"}
+                  />
+                  <label htmlFor="lastfm_import">
+                    <div className="title">Connect to Last.FM</div>
+                    <div className="details">
+                      We will periodically check your Last.FM account and add
+                      your new scrobbles to ListenBRainz
+                    </div>
+                  </label>
+                </button>
+                <ServicePermissionButton
+                  service="lastfm"
+                  current={permissions.lastFm ?? "disable"}
+                  value="disable"
+                  title="Disable"
+                  details="New scrobbles won't be imported from Last.FM"
+                  handlePermissionChange={handlePermissionChange}
+                />
+              </div>
+            </form>
+          </div>
+        </div>
+
         <div className="panel panel-default">
           <div className="panel-heading">
             <h3 className="panel-title">SoundCloud</h3>

frontend/js/src/settings/music-services/details/components/ExternalServiceButton.tsx

@@ -1,7 +1,12 @@
 import * as React from "react";
 
 type ExternalServiceButtonProps = {
-  service: "spotify" | "soundcloud" | "critiquebrainz" | "appleMusic";
+  service:
+    | "spotify"
+    | "soundcloud"
+    | "critiquebrainz"
+    | "appleMusic"
+    | "lastfm";
   current: string;
   value: string;
   title: string;

listenbrainz/db/external_service_oauth.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 from typing import List, Optional, Union
 
 from sqlalchemy import text
@@ -7,8 +8,9 @@
 import sqlalchemy
 
 
-def save_token(db_conn, user_id: int, service: ExternalServiceType, access_token: str, refresh_token: Optional[str],
-               token_expires_ts: int, record_listens: bool, scopes: List[str], external_user_id: Optional[str] = None):
+def save_token(db_conn, user_id: int, service: ExternalServiceType, access_token: Optional[str], refresh_token: Optional[str],
+               token_expires_ts: Optional[int], record_listens: bool, scopes: Optional[List[str]], external_user_id: Optional[str] = None,
+               latest_listened_at: Optional[datetime] = None):
     """ Add a row to the external_service_oauth table for specified user with corresponding tokens and information.
 
     Args:
@@ -21,6 +23,7 @@ def save_token(db_conn, user_id: int, service: ExternalServiceType, access_token
         record_listens: True if user wishes to import listens, False otherwise
         scopes: the oauth scopes
         external_user_id: the user's id in the external linked service
+        latest_listened_at: last listen import time
     """
     # regardless of whether a row is inserted or updated, the end result of the query
     # should remain the same. if not so, weird things can happen as it is likely we
@@ -29,7 +32,7 @@ def save_token(db_conn, user_id: int, service: ExternalServiceType, access_token
     # to use the new values. any column which does not have a new value to be set should
     # be explicitly set to the default value (which would have been used if the row was
     # inserted instead).
-    token_expires = utils.unix_timestamp_to_datetime(token_expires_ts)
+    token_expires = utils.unix_timestamp_to_datetime(token_expires_ts) if token_expires_ts else None
     result = db_conn.execute(sqlalchemy.text("""
         INSERT INTO external_service_oauth AS eso
         (user_id, external_user_id, service, access_token, refresh_token, token_expires, scopes)
@@ -58,20 +61,21 @@ def save_token(db_conn, user_id: int, service: ExternalServiceType, access_token
         external_service_oauth_id = result.fetchone().id
         db_conn.execute(sqlalchemy.text("""
             INSERT INTO listens_importer
-            (external_service_oauth_id, user_id, service)
+            (external_service_oauth_id, user_id, service, latest_listened_at)
             VALUES
-            (:external_service_oauth_id, :user_id, :service)
+            (:external_service_oauth_id, :user_id, :service, :latest_listened_at)
             ON CONFLICT (user_id, service) DO UPDATE SET
                 external_service_oauth_id = EXCLUDED.external_service_oauth_id,
                 user_id = EXCLUDED.user_id,
                 service = EXCLUDED.service,
                 last_updated = NULL,
-                latest_listened_at = NULL,
+                latest_listened_at = EXCLUDED.latest_listened_at,
                 error_message = NULL
             """), {
             "external_service_oauth_id": external_service_oauth_id,
             "user_id": user_id,
-            "service": service.value
+            "service": service.value,
+            "latest_listened_at": latest_listened_at,
         })
 
     db_conn.commit()