From 804d562dda10ed06ab088f2e81e9c5a550f16bde Mon Sep 17 00:00:00 2001 From: Jamesb Date: Sat, 24 Feb 2024 18:24:48 +0000 Subject: [PATCH] feat: clean up profile page and summary table --- .../client/src/components/AccountDropdown.tsx | 4 + packages/client/src/components/AdminPage.tsx | 29 ++-- packages/client/src/components/Homepage.tsx | 13 +- .../client/src/components/ProfilePage.tsx | 127 ++++++++++-------- .../client/src/components/SettingsModal.tsx | 77 +++++++++++ .../migration.sql | 5 + packages/server/prisma/schema.prisma | 2 +- packages/server/src/lib/jobsPrisma.ts | 20 +++ packages/server/src/routers/adminRouter.ts | 43 ++++-- packages/server/src/tasks/worker.ts | 21 +-- 10 files changed, 223 insertions(+), 118 deletions(-) create mode 100644 packages/client/src/components/SettingsModal.tsx create mode 100644 packages/server/prisma/migrations/20240224145230_delete_cascade_logs/migration.sql diff --git a/packages/client/src/components/AccountDropdown.tsx b/packages/client/src/components/AccountDropdown.tsx index 19623f9..0504459 100644 --- a/packages/client/src/components/AccountDropdown.tsx +++ b/packages/client/src/components/AccountDropdown.tsx @@ -4,6 +4,7 @@ import { logout } from "../lib/login"; import { useNavigate } from "react-router-dom"; import { z } from "zod"; import { UserModel } from "shared/src/schemas/user"; +import { SettingsModal } from "./SettingsModal"; interface AccountDropdownProps { user: z.infer; @@ -52,6 +53,9 @@ export const AccountDropdown = (props: AccountDropdownProps) => { > View Profile + + API Key + Logout diff --git a/packages/client/src/components/AdminPage.tsx b/packages/client/src/components/AdminPage.tsx index b7cc466..5f8526f 100644 --- a/packages/client/src/components/AdminPage.tsx +++ b/packages/client/src/components/AdminPage.tsx @@ -31,7 +31,7 @@ export const AdminPage = () => { // Fetch data immediately and then set up the interval fetchData(); - const interval = setInterval(fetchData, 2000); + const interval = setInterval(fetchData, 5000); // Clear interval on component unmount return () => clearInterval(interval); @@ -59,13 +59,8 @@ export const AdminPage = () => { Username - Created + Created At - { - // - // Status - // - } Tasks @@ -82,7 +77,9 @@ export const AdminPage = () => { {pipeline.username} - {dayjs(pipeline.createdAt).fromNow()} + + {dayjs(pipeline.createdAt).format("YY-MM-DD HH:mm:ss")} + { //{pipeline.status} } @@ -118,7 +115,15 @@ export const AdminPage = () => { {task.name} - {dayjs(task.createdAt).fromNow()} + { + // mins since pipeline created + "+" + + dayjs(task.createdAt).diff( + dayjs(pipeline.createdAt), + "minute" + ) + + " mins" + } {idx === 0 ? pipeline.status : task.status} @@ -129,11 +134,7 @@ export const AdminPage = () => { {task.logs .filter((x) => x.level !== "debug") .map((log, idx) => ( -
  • - {dayjs(log.createdAt).fromNow()} - {" - "} - {log.log} -
    • +
  • {log.log}
    • ))} )} diff --git a/packages/client/src/components/Homepage.tsx b/packages/client/src/components/Homepage.tsx index e2e548c..700ce1b 100644 --- a/packages/client/src/components/Homepage.tsx +++ b/packages/client/src/components/Homepage.tsx @@ -51,18 +51,7 @@ export function Homepage(props: HomepageProps) {



    -

    Changelog

    -
      -
    • 2024-01-21: Implement user profile and follower system.
      • -
    • 2024-01-14: Add API.
      • -
    • 2024-01-13: Add auth and server.
      • -
    • - 2024-01-08: Integrate metaphor - . -
      • -
    -

    -

    Top Users:

    +

    Example Users:

      {(users || []).map((user) => (
    • diff --git a/packages/client/src/components/ProfilePage.tsx b/packages/client/src/components/ProfilePage.tsx index 2ff3355..b2da705 100644 --- a/packages/client/src/components/ProfilePage.tsx +++ b/packages/client/src/components/ProfilePage.tsx @@ -6,6 +6,7 @@ import { RouterOutput, trpc } from "../lib/trpc"; import { Avatar, Button, + Collapse, Paper, Table, TableBody, @@ -13,10 +14,14 @@ import { TableContainer, TableHead, TableRow, + TextField, Tooltip, + Typography, } from "@mui/material"; import * as _ from "remeda"; -import { OnboardingModal } from "./OnboardingModal"; +import dayjs from "dayjs"; +import ExpandMoreIcon from "@mui/icons-material/ExpandMore"; +import ExpandLessIcon from "@mui/icons-material/ExpandLess"; interface ProfilePageProps { auth: AuthInfo | undefined; @@ -84,10 +89,7 @@ export function ProfilePage(props: ProfilePageProps) { setProfileForUser(response); }); }, [props.auth, usernameForProfile]); - const [apiKey, setApiKey] = React.useState<{ - loading: boolean; - data?: string; - }>(); + if (!props.auth?.authenticated) { return (
      @@ -100,7 +102,7 @@ export function ProfilePage(props: ProfilePageProps) { summaries?.forEach((summary) => { rows.push({ createdAt: summary.createdAt, - type: "Summary of User", + type: "Twitter Summary", content: summary.content, useForRecommendations: summary.useForRecommendations, }); @@ -147,61 +149,26 @@ export function ProfilePage(props: ProfilePageProps) { {isFollowing ? "Unfollow" : "Follow"} - ) : ( - <> - {!apiKey ? ( - - ) : ( -
      - Your API Key ( - - What is this? - - ): 
      {apiKey.data}
      -
      - )} - - )} + ) : null}


      -
      This table summarizes recommendation inputs.
      -

      - { - - - - } + +
      + + +


      +

      Recommendation Inputs



      - Created At + Date Type Content {viewingOwnProfile && Actions} @@ -209,12 +176,11 @@ export function ProfilePage(props: ProfilePageProps) { {_.sortBy(rows, (x) => x.createdAt).map((row, idx) => ( - - {row.createdAt} - {row.type} - {row.content} - {viewingOwnProfile && Edit} - + ))}
      @@ -222,3 +188,46 @@ export function ProfilePage(props: ProfilePageProps) { ); } + +interface SummaryRowProps { + row: TableRow; + viewingOwnProfile: boolean; +} + +function SummaryRow(props: SummaryRowProps) { + const { row, viewingOwnProfile } = props; + const [expanded, setExpanded] = React.useState(false); + return ( + + {dayjs(row.createdAt).format("YY-MM-DD")} + {row.type} + + + {row.content} +
      { + setExpanded(!expanded); + }} + > + {expanded ? : } +
      +       +       + {viewingOwnProfile && ( + +
      {/* */}
      +
      + +
      +       + )} +       + ); +} diff --git a/packages/client/src/components/SettingsModal.tsx b/packages/client/src/components/SettingsModal.tsx new file mode 100644 index 0000000..d67960a --- /dev/null +++ b/packages/client/src/components/SettingsModal.tsx @@ -0,0 +1,77 @@ +import { + Button, + Dialog, + DialogActions, + DialogContent, + DialogTitle, +} from "@mui/material"; +import React from "react"; +import { trpc } from "../lib/trpc"; +import { OnboardingModal } from "./OnboardingModal"; + +interface SettingsModalProps { + children: React.ReactNode; +} + +export function SettingsModal(props: SettingsModalProps) { + const [isOpen, setOpen] = React.useState(false); + const handleOpen = () => setOpen(true); + const handleClose = () => setOpen(false); + const [apiKey, setApiKey] = React.useState<{ + loading: boolean; + data?: string; + }>(); + return ( + <> + { + e.stopPropagation(); + e.preventDefault(); + handleOpen(); + }} + > + {props.children} + + + Settings + +
      + {!apiKey ? ( + + ) : ( +
      + Your API Key ( + + What is this? + + ): 
      {apiKey.data}
      +
      + )} +
      +       + + + +       + + ); +} diff --git a/packages/server/prisma/migrations/20240224145230_delete_cascade_logs/migration.sql b/packages/server/prisma/migrations/20240224145230_delete_cascade_logs/migration.sql new file mode 100644 index 0000000..068743a --- /dev/null +++ b/packages/server/prisma/migrations/20240224145230_delete_cascade_logs/migration.sql @@ -0,0 +1,5 @@ +-- DropForeignKey +ALTER TABLE "PipelineTaskLog" DROP CONSTRAINT "PipelineTaskLog_pipelineTaskId_fkey"; + +-- AddForeignKey +ALTER TABLE "PipelineTaskLog" ADD CONSTRAINT "PipelineTaskLog_pipelineTaskId_fkey" FOREIGN KEY ("pipelineTaskId") REFERENCES "PipelineTask"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/packages/server/prisma/schema.prisma b/packages/server/prisma/schema.prisma index e56fe45..cdc0e69 100644 --- a/packages/server/prisma/schema.prisma +++ b/packages/server/prisma/schema.prisma @@ -223,7 +223,7 @@ model PipelineTaskLog { createdAt DateTime @default(now()) updatedAt DateTime @updatedAt pipelineTaskId Int - pipelineTask PipelineTask @relation(fields: [pipelineTaskId], references: [id]) + pipelineTask PipelineTask @relation(fields: [pipelineTaskId], references: [id], onDelete: Cascade) level String log String } diff --git a/packages/server/src/lib/jobsPrisma.ts b/packages/server/src/lib/jobsPrisma.ts index 0e75f39..9f65b21 100644 --- a/packages/server/src/lib/jobsPrisma.ts +++ b/packages/server/src/lib/jobsPrisma.ts @@ -20,3 +20,23 @@ export async function getPipelineTaskJobById(id: string): Promise<{ console.log("getPipelineTaskJobById", raw); return raw[0]; } + +interface Job { + id: number; + queue_name: string | null; + task_identifier: string; + priority: number; + run_at: Date; + attempts: number; + max_attempts: number; + last_error: string | null; + created_at: Date; + updated_at: Date; + key: string | null; +} + +export async function getGraphileJobs(): Promise { + return await prisma.$queryRaw( + Prisma.sql`SELECT * FROM graphile_worker.jobs` + ); +} diff --git a/packages/server/src/routers/adminRouter.ts b/packages/server/src/routers/adminRouter.ts index 7d9c983..f798241 100644 --- a/packages/server/src/routers/adminRouter.ts +++ b/packages/server/src/routers/adminRouter.ts @@ -8,7 +8,12 @@ import { } from "../lib/addRecomendations"; import { getSavedTweetsForUser } from "../lib/tweets"; import { workerUtils } from "../tasks/workerUtils"; -import { getPipelineJobByKey, getPipelineTaskJobById } from "../lib/jobsPrisma"; +import { + getGraphileJobs, + getPipelineJobByKey, + getPipelineTaskJobById, +} from "../lib/jobsPrisma"; +import { indexBy } from "remeda"; export const adminRouter = router({ addRecommendations: publicProcedure @@ -88,15 +93,25 @@ export const adminRouter = router({ return user; }), getPipelinesAndTasks: publicProcedure.query(async ({ ctx }) => { - const pipelines = await prisma.pipelineRun.findMany({ - include: { - tasks: { - include: { - logs: true, + const jobs = indexBy(await getGraphileJobs(), (x) => x.key || x.id); + const pipelines = ( + await prisma.pipelineRun.findMany({ + include: { + tasks: { + include: { + logs: true, + }, }, }, - }, - }); + }) + ).map((p) => ({ + ...p, + tasks: p.tasks.map((t) => ({ + ...t, + job: jobs[t.jobId], + })), + job: jobs[p.jobKeyId], + })); return pipelines; }), retryPipelineTask: publicProcedure @@ -171,11 +186,13 @@ export const adminRouter = router({ } const utils = await workerUtils(); const pipelineJob = await getPipelineJobByKey(pipeline.jobKeyId); - const failed = await utils.permanentlyFailJobs([ - pipelineJob.id.toString(), - ...pipeline.tasks.map((t) => t.jobId.toString()), - ]); - console.log("successfully failed jobs", failed); + if (pipelineJob) { + const failed = await utils.permanentlyFailJobs([ + pipelineJob.id.toString(), + ...pipeline.tasks.map((t) => t.jobId.toString()), + ]); + console.log("successfully failed", failed.length, "jobs"); + } await prisma.pipelineRun.delete({ where: { id: input.id, diff --git a/packages/server/src/tasks/worker.ts b/packages/server/src/tasks/worker.ts index b0b4b48..54f3f90 100644 --- a/packages/server/src/tasks/worker.ts +++ b/packages/server/src/tasks/worker.ts @@ -18,6 +18,7 @@ export const addPipeline = async >( payload: TaskNamePayloadMaps[Name] & { username: string; runId: string; + runAt?: Date; } ) => { console.log("Adding pipeline", taskName, payload); @@ -30,6 +31,7 @@ export const addPipeline = async >( }); const job = await utils.addJob(taskName, payload as any, { jobKey: payload.runId, + runAt: payload.runAt, }); return job; }; @@ -121,22 +123,3 @@ export async function startWorker() { } }); } - -if (require.main === module) { - (async () => { - await startWorker(); - // await new Promise((resolve) => setTimeout(resolve, 5000)); - // const summary = `Jamesb, a user with a fascination for Large Language Models (LLMs) and Spaced Repetition Systems (SRS), has shown a deep interest in understanding and improving learning methods, particularly in merging the worlds of recommender systems and flashcard learning applications, as shown by their tweets and article 'From Spaced Repetition Systems to Open Recommender Systems'. Jamesb enjoys engaging discussions and shares on subjects such as open-source artificial intelligence (AI), LLMs, technology restrictions, and perception of average individuals, evident from their interactions with users like Zander and Lars_Grammel. They also have a liking for podcasts, specifically ones by Joe Walker and Dwarkesh Patel, proving an interest in intellectual conversations and exploring novel ideas. They are involved in the project titled 'Twitter for Free Learners', working with @norman.nashwin while aspiring to post more on Twitter and interact with the Internet learning community. Their support for RSS reader-style internet consumption indicates an enthusiasm for personalized content recommendation. Himself/herself is building an app under the name 'rem_note' in the multiverse. Jamesb, a technology enthusiast with a passion for artificial intelligence (AI), primarily focuses on the development and application of large language models (LLMs) and Spaced Repetition Systems (SRS). He applies these interests to build recommender systems, believing that such systems should focus on providing timeless and personalized content. These systems involve using metadata created by Generative Pretrained Transformer 2 (GPT-2) models and techniques such as pipeline abstractions or memoize functions for debugging. He is currently building an SRS application called RemNote and is intrigued by how OpenAI’s text-to-speech (TTS) technology could be integrated into his developments. Jamesb also explores the utilisation of LLMs in citation validation, specific methods of software development, and AI for music generation. Active in the tech community and a supporter of the open-source approach, Jamesb interacts with technologists like Erik Bjäreholt and Kyle Corbitt, and engages in public discourse around companies like OpenAI. In conjunction with his technological interests, he also maintains an active interest in AI research and social network innovation, with a particular focus on French innovation in these fields. Jamesb is not only an ardent supporter of technological developments but also promotes the ideological shifts to make these technologies more efficient and user-centric. He supports the ideas of Dominic Cummings and David Deutsch, notably the restructuring of Western governments. He is also a key promoter of the quick iteration process, as he believes in the elimination of slow feedback loops to foster dynamic progress. Jamesb, an artificial intelligence (AI) enthusiast with particular interest in large language models (LLMs), is currently working on building a space repetition system application named Remnote. Recently, he has been actively tweeting about leveraging tools like the BakLLaVA-1 on Huggingface and GPT-4V (probable successor to OpenAI's GPT-3) to help build AI systems, especially for web scraping tasks. As a part of his work, he is experimenting with a retrieval augmented generation (RAG) to create an AI journaling assistant. This assistant extracts metadata from paragraphs such as emotions, people, and topics mentioned using GPT-4V and then summarises the paragraphs for quick memory extraction whenever a new entry is written. He also seems interested in exploring how LLMs can interact with browser extensions like Vimium for selecting screen elements, bypassing the need for mouse positions or passing the browser DOM (Document Object Model) as inputs. Additionally, he has shown interest in understanding the intricacies of model-training and metadata extraction from GPT, further evidenced by his interaction with other AI practitioners such as Matthew Barlow, Rob Haisfield, and Daniel George. While appreciating the improvements in RAG's accuracy mentioned by Ben Parr, James indicated interest in any planned updates for assistants' API regarding retrieval strategies. Jamesb is an enthusiast of large language models (LLMs), often exploring their applications in building natural language interfaces, voice cloning, and automated meal planning. His projects include a chat interface using transcripts of YouTuber Brett Hall and David Deutsch's cloned voice created with ElevenLabs, and a customised meal plan project, which assembles ingredients and adds them to a grocery cart. He also engages with work by creators like Arjun_Khemani, showcasing his interest in speech synthesis services like LMNT and ElevenLabs, and the EU Cyber Resilience Act. He further extensively works on Spaced Repetition Systems (SRS), particularly when transforming these systems into engaging learning experiences. As a software developer, he enjoys coding for automations and has built the open-source project 'Open Fresh', which leverages artificial intelligence to prepare custom meal plans. James underscores the necessity of caution in the development of Artificial General Intelligence (AGI). He also takes part in discussions related to AI regulation and progress, Web scraping using selenium and LLMs, and GitHub Copilot. Jamesb enjoys sharing his work on GitHub and engaging in insightful discourse with the wider development community and AI enthusiasts. Jamesb is a software developer who is primarily interested in exploring the use of large language models (LLMs) for the development of comprehensive, interactive learning systems. His focus is on the Spaced Repetition System (SRS), particularly its application for enhancing learning experiences in various domains such as mathematics and software development. In addition to building SRS applications, Jamesb has made notable contributions to the development of innovative learning tools, with features such as incremental reading, writing, and video integration into RemNote, a digital note-taking app. Besides, his work stands out in the integration of artificial intelligence assistants for reflective journaling and the use of metaphoric systems for recommending new content. Acknowledging his fondness for Lean, a math-proof oriented programming language, Jamesb has developed exercise templates and a Lean plugin for RemNote, essentially modifying it to a digital tutor for math proofs. The exercise templates in conjunction with a robust SRS framework are part of his contribution to creating a more adaptive learning environment. Beyond his work, Jamesb actively engages with the technology and AI community where he shares and discusses advancements in artificial intelligence, with particular interest in the development and implications surrounding GPT4 (Generative Pretrained Transformer 4). He is curious about the views industry thought leaders such as George Hotz, Melanie Mitchell, Michael Nielsen, and Douglas Hofstadter have on areas like AI risk and GPT4. Among his shared resources and content, there's also an evident interest in mathematics, reflected by his shared articles on introductory level mathematics principles and proofs. Jamesb's wide array of interests extends from building advanced learning systems to actively participating in conversations on emerging AI technologies while also expressing passion for educating students on mathematics principles and proofs.`; - // const customQuery = `How to solve sleep maintenance insomnia?`; - // const username = `experilearning`; - // const pipelineJobKey = randomUUID(); - // const job = await addPipeline("twitter-pipeline-v1", { - // summary, - // queries: [customQuery], - // username, - // runId: pipelineJobKey, - // emailResults: true, - // }); - // })(); - })(); -}