From 1de7f793e70eddbbae6664c0b7b95f491f12efa5 Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Sun, 19 Dec 2021 19:36:08 +0900 Subject: [PATCH 01/23] :package: update: swift5.5 --- Dockerfile | 6 +++--- Package.swift | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Dockerfile b/Dockerfile index eff40f4..6d757b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ # ================================ # Develop image # ================================ -FROM swift:5.3-focal as develop +FROM swift:5.5-focal as develop # Install OS updates and, if needed, sqlite3 RUN export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true \ @@ -37,7 +37,7 @@ RUN [ -d /develop/Resources ] && { mv /develop/Resources ./Resources && chmod -R # ================================ # Run image # ================================ -FROM swift:5.3-focal-slim +FROM swift:5.5-focal-slim # Make sure all system packages are up to date. RUN export DEBIAN_FRONTEND=noninteractive DEBCONF_NONINTERACTIVE_SEEN=true && \ @@ -60,4 +60,4 @@ EXPOSE 8080 # Start the Vapor service when the image is run, default to listening on 8080 in production environment ENTRYPOINT ["./Run"] -CMD ["serve", "--env", "production", "--hostname", "0.0.0.0", "--port", "8080"] \ No newline at end of file +CMD ["serve", "--env", "production", "--hostname", "0.0.0.0", "--port", "8080"] diff --git a/Package.swift b/Package.swift index a5a1534..e2be889 100644 --- a/Package.swift +++ b/Package.swift @@ -1,10 +1,10 @@ -// swift-tools-version:5.2 +// swift-tools-version:5.5 import PackageDescription let package = Package( name: "agqr-program-guide", platforms: [ - .macOS(.v10_15) + .macOS(.v12) ], dependencies: [ // 💧 A server-side Swift web framework. From d280f7bd4d577f4341d468e118da6f95986c4cc7 Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Sun, 19 Dec 2021 19:38:56 +0900 Subject: [PATCH 02/23] =?UTF-8?q?:wrench:=20fix:=20=E5=85=AC=E5=BC=8F?= =?UTF-8?q?=E3=81=AB=E6=B2=BF=E3=81=A3=E3=81=9Ftls=E3=81=AE=E8=A8=AD?= =?UTF-8?q?=E5=AE=9A?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Sources/App/configure.swift | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Sources/App/configure.swift b/Sources/App/configure.swift index a37b1c5..912ea99 100644 --- a/Sources/App/configure.swift +++ b/Sources/App/configure.swift @@ -9,7 +9,8 @@ import Vapor public func configure(_ app: Application) throws { // uncomment to serve files from /Public folder // app.middleware.use(FileMiddleware(publicDirectory: app.directory.publicDirectory)) - + var tls = TLSConfiguration.makeClientConfiguration() + tls.certificateVerification = .none app.databases.use( .mysql( hostname: Environment.get("DATABASE_HOST") ?? "localhost", @@ -17,7 +18,7 @@ public func configure(_ app: Application) throws { username: Environment.get("DATABASE_USERNAME") ?? "vapor_username", password: Environment.get("DATABASE_PASSWORD") ?? "vapor_password", database: Environment.get("DATABASE_NAME") ?? "vapor_database", - tlsConfiguration: .forClient(certificateVerification: .none) + tlsConfiguration: tls ), as: .mysql) app.migrations.add(CreateProgram()) From ece8738ae23b2ffa0f22881128f4c1f228bef16d Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Sun, 19 Dec 2021 19:41:10 +0900 Subject: [PATCH 03/23] =?UTF-8?q?:sparkles:=20feat:=20async/await=E5=AF=BE?= =?UTF-8?q?=E5=BF=9C=E7=89=88(=E6=9A=AB=E5=AE=9A)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Sources/App/Jobs/ImportProgramGuideJob.swift | 29 ++++--- .../Repositories/ProgramGuideRepository.swift | 76 +++++++++---------- .../Services/DownloadAgqrProgramGuide.swift | 36 ++++++--- 3 files changed, 76 insertions(+), 65 deletions(-) diff --git a/Sources/App/Jobs/ImportProgramGuideJob.swift b/Sources/App/Jobs/ImportProgramGuideJob.swift index 75e70f8..2e2b8a3 100644 --- a/Sources/App/Jobs/ImportProgramGuideJob.swift +++ b/Sources/App/Jobs/ImportProgramGuideJob.swift @@ -13,17 +13,24 @@ struct ImportProgramGuideJob: ScheduledJob { defer { context.logger.info("End Scraping Process") } - - return client.fetchWeekly(app: context.application).flatMap { responses -> EventLoopFuture in - responses - .map { response -> EventLoopFuture in - guard let response = response else { - return context.application.eventLoopGroup.future(error: "番組表データがありませんでした。") - } - let programGuide = self.parser.parse(response) - return self.repository.save(programGuide, app: context.application) - } - .flatten(on: context.application.eventLoopGroup.next()) + let promise = context.eventLoop.makePromise(of: Void.self) + promise.completeWithTask { + await self.asyncRun(context: context) + } + return promise.futureResult + + } + + func asyncRun(context: QueueContext) async -> Void { + let responses = await client.fetchWeekly(app: context.application) + for response in responses { + guard let response = response else { + // TODO: ログに変更 + print("番組表データがありません。") + continue + } + let programGuide = self.parser.parse(response) + await self.repository.save(programGuide, app: context.application) } } } diff --git a/Sources/App/Repositories/ProgramGuideRepository.swift b/Sources/App/Repositories/ProgramGuideRepository.swift index 660bfdf..c14ccdd 100644 --- a/Sources/App/Repositories/ProgramGuideRepository.swift +++ b/Sources/App/Repositories/ProgramGuideRepository.swift @@ -2,67 +2,59 @@ import Fluent import Vapor protocol ProgramGuideSaving { - func save(_ items: [ProgramGuide], app: Application) -> EventLoopFuture + func save(_ items: [ProgramGuide], app: Application) async -> Void } struct ProgramGuideRepository: ProgramGuideSaving { - func save(_ items: [ProgramGuide], app: Application) -> EventLoopFuture { - // 与えられたProgramGuideからProgramとPersonalitiesをDBに保存し、リレーションを貼る - let res = items.map { item -> EventLoopFuture in - // programを取得し、存在する場合は更新する - let program = upsertProgram(item.program, app.db) - let personalities = item.personalities.map { upsertPersonality($0, app.db) } - // 両方の成功時にリレーションを貼る - return personalities.map { - $0.flatMap { personality -> EventLoopFuture in - program.flatMap { program -> EventLoopFuture in - program.$personalities.isAttached(to: personality, on: app.db).flatMap { - isAttach -> EventLoopFuture in - if !isAttach { - return program.$personalities.attach(personality, on: app.db) - } - return app.eventLoopGroup.future() - } + func save(_ items: [ProgramGuide], app: Application) async -> Void { + for programGuide in items { + do { + let insertedProgram = try await upsertProgram(programGuide.program, app.db) + var insertedPersonalities: [Personality] = [] + for personality in programGuide.personalities { + let p = try await upsertPersonality(personality, app.db) + insertedPersonalities.append(p) + } + for personality in insertedPersonalities { + let isAttached = try? await insertedProgram.$personalities.isAttached(to: personality, on: app.db) + if isAttached == false { + try await insertedProgram.$personalities.attach(personality, on: app.db) } } - } - .flatten(on: app.eventLoopGroup.next()) - .flatMapError { error in + } catch { + print("error") print(error.localizedDescription) - return app.eventLoopGroup.future(error: error) } } - return res.flatten(on: app.eventLoopGroup.next()) } /// 開始時間と終了時間を基準に(unique)insert or updateを行う. - func upsertProgram(_ program: Program, _ db: Database) -> EventLoopFuture { - return - Program - .query(on: db) + func upsertProgram(_ program: Program, _ db: Database) async throws -> Program { + let targetProgram = try? await Program.query(on: db) .filter(\.$startDatetime == program.startDatetime) .filter(\.$endDatetime == program.endDatetime) .first() - .flatMap { dbProgram -> EventLoopFuture in - program.id = dbProgram?.id - // idを挿入するだけだとうまくupdateできなかったため、判定要素を上書きする - program._$id.exists = dbProgram?.id != nil - return program.save(on: db).transform(to: program) - } + + program.id = targetProgram?.id + // idを挿入するだけだとうまくupdateできなかったため、判定要素を上書きする + program._$id.exists = targetProgram != nil + + try await program.save(on: db) + return program } /// 名前を基準に(unique)insert or updateを行う. - func upsertPersonality(_ personality: Personality, _ db: Database) -> EventLoopFuture { - return - Personality + func upsertPersonality(_ personality: Personality, _ db: Database) async throws -> Personality { + let targetPersonality = try? await Personality .query(on: db) .filter(\.$name == personality.name) .first() - .flatMap { dbPersonality -> EventLoopFuture in - personality.id = dbPersonality?.id - // idを挿入するだけだとうまくupdateできなかったため、判定要素を上書きする - personality._$id.exists = dbPersonality?.id != nil - return personality.save(on: db).transform(to: personality) - } + + personality.id = targetPersonality?.id + // idを挿入するだけだとうまくupdateできなかったため、判定要素を上書きする + personality._$id.exists = targetPersonality != nil + + try await personality.save(on: db) + return personality } } diff --git a/Sources/App/Services/DownloadAgqrProgramGuide.swift b/Sources/App/Services/DownloadAgqrProgramGuide.swift index 87403d5..007c1ab 100644 --- a/Sources/App/Services/DownloadAgqrProgramGuide.swift +++ b/Sources/App/Services/DownloadAgqrProgramGuide.swift @@ -4,13 +4,11 @@ struct DownloadAgqrProgramGuide { // swift-format-ignore static let AGQR_URL = "https://www.joqr.co.jp/qr/agdailyprogram/" - func fetchToday(app: Application) -> EventLoopFuture { - return app.client.get(URI(string: Self.AGQR_URL)).map { res -> Data? in - return res.body.flatMap { $0.getData(at: 0, length: $0.writerIndex) } - } + func fetchToday(app: Application) async -> Data? { + return await self.execute(app: app, url: Self.AGQR_URL) } - func fetchWeekly(app: Application) -> EventLoopFuture<[Data?]> { + func fetchWeekly(app: Application) async -> [Data?] { // query date format=yyyyMMdd let formatter = DateFormatter() formatter.locale = Locale(identifier: "en_US_POSIX") @@ -21,16 +19,30 @@ struct DownloadAgqrProgramGuide { return formatter.string(from: calculateDate) }.map { "\(Self.AGQR_URL)?date=\($0)" } - return urls.map { url -> EventLoopFuture in - app.client.get(URI(string: url)).map { res -> Data? in - return res.body.flatMap { $0.getData(at: 0, length: $0.writerIndex) } + var responses: [Data?] = [] + await withTaskGroup(of: Data?.self) { group in + for url in urls { + group.addTask { + return await self.execute(app: app, url: url) + } + } + + for await response in group { + responses.append(response) } - }.flatten(on: app.eventLoopGroup.next()) + } + return responses } - func execute(app: Application, url: String?) -> EventLoopFuture { - return app.client.get(URI(string: url ?? Self.AGQR_URL)).map { res in - return res.body.flatMap { $0.getData(at: 0, length: $0.writerIndex) } + func execute(app: Application, url: String) async -> Data? { + do { + let response = try await app.client.get(URI(string: url)).get() + return response.body.flatMap { buffer in + buffer.getData(at: 0, length: buffer.writerIndex) + } + } catch { + print(error.localizedDescription) + return nil } } } From e6ab3bcf58501bba8841798fb485887d24bb10e0 Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Sun, 19 Dec 2021 19:41:47 +0900 Subject: [PATCH 04/23] =?UTF-8?q?:bug:=20fix:=20=E5=8B=95=E4=BD=9C?= =?UTF-8?q?=E7=A2=BA=E8=AA=8D=E7=94=A8=E3=81=AE=E3=82=B3=E3=83=9E=E3=83=B3?= =?UTF-8?q?=E3=83=89=E3=82=92=E5=BE=AE=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../App/Commands/ScrapingAgqrCommand.swift | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/Sources/App/Commands/ScrapingAgqrCommand.swift b/Sources/App/Commands/ScrapingAgqrCommand.swift index 63dbd8b..3eefe6f 100644 --- a/Sources/App/Commands/ScrapingAgqrCommand.swift +++ b/Sources/App/Commands/ScrapingAgqrCommand.swift @@ -7,8 +7,8 @@ struct ScrapingAgqr: Command { let client = DownloadAgqrProgramGuide() struct Signature: CommandSignature { - @Option(name: "url") - var url: String? + @Argument(name: "url") + var url: String } var help: String = "Download program guide and parse to json." @@ -18,18 +18,22 @@ struct ScrapingAgqr: Command { defer { context.console.info("End Process") } - let future = client.execute(app: context.application, url: signature.url) - .unwrap(or: fatalError("htmlデータの取得に失敗しました")) - .flatMap { res -> EventLoopFuture in - let programGuide = self.parser.parse(res) - return repository.save(programGuide, app: context.application) - } - - do { - try future.wait() - } catch { - print("Batch failure") - print(error.localizedDescription) + + let promise = context.application.eventLoopGroup.next().makePromise(of: Void.self) + promise.completeWithTask { + await self.asyncRun(using: context, signature: signature) + } + + try promise.futureResult.wait() + } + + func asyncRun(using context: CommandContext, signature: Signature) async { + let response = await client.execute(app: context.application, url: signature.url) + guard let response = response else { + print("htmlデータの取得に失敗しました") + return } + let programGuide = parser.parse(response) + await repository.save(programGuide, app: context.application) } } From 0f4632a22063cd2b0c6ef000ca44d840919f63b0 Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Wed, 29 Dec 2021 12:58:34 +0900 Subject: [PATCH 05/23] :recycle: refactor: swift-format --- Sources/App/Commands/ScrapingAgqrCommand.swift | 6 +++--- Sources/App/Jobs/ImportProgramGuideJob.swift | 5 ++--- .../App/Repositories/ProgramGuideRepository.swift | 14 ++++++++------ .../App/Services/DownloadAgqrProgramGuide.swift | 2 +- Sources/App/configure.swift | 2 +- tools/Package.resolved | 12 ++++++------ tools/Package.swift | 4 ++-- 7 files changed, 23 insertions(+), 22 deletions(-) diff --git a/Sources/App/Commands/ScrapingAgqrCommand.swift b/Sources/App/Commands/ScrapingAgqrCommand.swift index 3eefe6f..36ed7e5 100644 --- a/Sources/App/Commands/ScrapingAgqrCommand.swift +++ b/Sources/App/Commands/ScrapingAgqrCommand.swift @@ -18,15 +18,15 @@ struct ScrapingAgqr: Command { defer { context.console.info("End Process") } - + let promise = context.application.eventLoopGroup.next().makePromise(of: Void.self) promise.completeWithTask { await self.asyncRun(using: context, signature: signature) } - + try promise.futureResult.wait() } - + func asyncRun(using context: CommandContext, signature: Signature) async { let response = await client.execute(app: context.application, url: signature.url) guard let response = response else { diff --git a/Sources/App/Jobs/ImportProgramGuideJob.swift b/Sources/App/Jobs/ImportProgramGuideJob.swift index 2e2b8a3..ccc3890 100644 --- a/Sources/App/Jobs/ImportProgramGuideJob.swift +++ b/Sources/App/Jobs/ImportProgramGuideJob.swift @@ -18,10 +18,9 @@ struct ImportProgramGuideJob: ScheduledJob { await self.asyncRun(context: context) } return promise.futureResult - } - - func asyncRun(context: QueueContext) async -> Void { + + func asyncRun(context: QueueContext) async { let responses = await client.fetchWeekly(app: context.application) for response in responses { guard let response = response else { diff --git a/Sources/App/Repositories/ProgramGuideRepository.swift b/Sources/App/Repositories/ProgramGuideRepository.swift index c14ccdd..543cafb 100644 --- a/Sources/App/Repositories/ProgramGuideRepository.swift +++ b/Sources/App/Repositories/ProgramGuideRepository.swift @@ -2,11 +2,11 @@ import Fluent import Vapor protocol ProgramGuideSaving { - func save(_ items: [ProgramGuide], app: Application) async -> Void + func save(_ items: [ProgramGuide], app: Application) async } struct ProgramGuideRepository: ProgramGuideSaving { - func save(_ items: [ProgramGuide], app: Application) async -> Void { + func save(_ items: [ProgramGuide], app: Application) async { for programGuide in items { do { let insertedProgram = try await upsertProgram(programGuide.program, app.db) @@ -16,7 +16,8 @@ struct ProgramGuideRepository: ProgramGuideSaving { insertedPersonalities.append(p) } for personality in insertedPersonalities { - let isAttached = try? await insertedProgram.$personalities.isAttached(to: personality, on: app.db) + let isAttached = try? await insertedProgram.$personalities.isAttached( + to: personality, on: app.db) if isAttached == false { try await insertedProgram.$personalities.attach(personality, on: app.db) } @@ -38,18 +39,19 @@ struct ProgramGuideRepository: ProgramGuideSaving { program.id = targetProgram?.id // idを挿入するだけだとうまくupdateできなかったため、判定要素を上書きする program._$id.exists = targetProgram != nil - + try await program.save(on: db) return program } /// 名前を基準に(unique)insert or updateを行う. func upsertPersonality(_ personality: Personality, _ db: Database) async throws -> Personality { - let targetPersonality = try? await Personality + let targetPersonality = + try? await Personality .query(on: db) .filter(\.$name == personality.name) .first() - + personality.id = targetPersonality?.id // idを挿入するだけだとうまくupdateできなかったため、判定要素を上書きする personality._$id.exists = targetPersonality != nil diff --git a/Sources/App/Services/DownloadAgqrProgramGuide.swift b/Sources/App/Services/DownloadAgqrProgramGuide.swift index 007c1ab..70d52cd 100644 --- a/Sources/App/Services/DownloadAgqrProgramGuide.swift +++ b/Sources/App/Services/DownloadAgqrProgramGuide.swift @@ -26,7 +26,7 @@ struct DownloadAgqrProgramGuide { return await self.execute(app: app, url: url) } } - + for await response in group { responses.append(response) } diff --git a/Sources/App/configure.swift b/Sources/App/configure.swift index 912ea99..73bd112 100644 --- a/Sources/App/configure.swift +++ b/Sources/App/configure.swift @@ -46,7 +46,7 @@ public func configure(_ app: Application) throws { allowedOrigin: .all, allowedMethods: [.GET, .OPTIONS, .HEAD], allowedHeaders: [ - .accept, .contentType, .origin, .xRequestedWith, .userAgent, .accessControlAllowOrigin + .accept, .contentType, .origin, .xRequestedWith, .userAgent, .accessControlAllowOrigin, ] ) app.middleware.use(CORSMiddleware(configuration: corsConfiguration)) diff --git a/tools/Package.resolved b/tools/Package.resolved index d6a8022..abc38d1 100644 --- a/tools/Package.resolved +++ b/tools/Package.resolved @@ -6,16 +6,16 @@ "repositoryURL": "https://github.com/apple/swift-argument-parser.git", "state": { "branch": null, - "revision": "83b23d940471b313427da226196661856f6ba3e0", - "version": "0.4.4" + "revision": "e1465042f195f374b94f915ba8ca49de24300a0d", + "version": "1.0.2" } }, { "package": "swift-format", "repositoryURL": "https://github.com/apple/swift-format", "state": { - "branch": "swift-5.4-branch", - "revision": "9c15831b798d767c9af0927a931de5d557004936", + "branch": "swift-5.5-branch", + "revision": "f872223e16742fd97fabd319fbf4a939230cc796", "version": null } }, @@ -24,8 +24,8 @@ "repositoryURL": "https://github.com/apple/swift-syntax", "state": { "branch": null, - "revision": "2fff9fc25cdc059379b6bd309377cfab45d8520c", - "version": "0.50400.0" + "revision": "75e60475d9d8fd5bbc16a12e0eaa2cb01b0c322e", + "version": "0.50500.0" } } ] diff --git a/tools/Package.swift b/tools/Package.swift index 59a59d2..3cfdbc6 100644 --- a/tools/Package.swift +++ b/tools/Package.swift @@ -1,10 +1,10 @@ -// swift-tools-version:5.2 +// swift-tools-version:5.5 import PackageDescription let package = Package( name: "Tools", dependencies: [ - .package(url: "https://github.com/apple/swift-format", .branch("swift-5.4-branch")), + .package(url: "https://github.com/apple/swift-format", .branch("swift-5.5-branch")), // .package(url: "https://github.com/realm/SwiftLint.git", .upToNextMajor(from: "0.43.1")) ] ) From 075ce27fbd4cc57e9295bb2bd5a1f7db79ee82cc Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Fri, 6 May 2022 02:27:34 +0900 Subject: [PATCH 06/23] =?UTF-8?q?fix:=20=E3=83=9E=E3=83=88=E3=83=AA?= =?UTF-8?q?=E3=82=AF=E3=82=B9CI?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/test.yaml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index eafcd2a..513298f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -6,10 +6,20 @@ on: - master jobs: - setup: - runs-on: macos-latest + build: + name: Swift ${{ matrix.swift }} on ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + swift: ["5.3", "5.5"] + runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v2 + - uses: fwal/setup-swift@2040b795e5c453c3a05fcb8316496afc8a74f192 + with: + swift-version: ${{ matrix.swift }} + - uses: actions/checkout@v3 - run: swift package resolve - - name: swift test + - name: Build + run: swift build + - name: Run tests run: swift test From 7057e849ba96f8fc26c8f85f965443e56da8142b Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Fri, 6 May 2022 02:33:17 +0900 Subject: [PATCH 07/23] fixup --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 513298f..57a046a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -11,7 +11,7 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] - swift: ["5.3", "5.5"] + swift: ["5.5"] runs-on: ${{ matrix.os }} steps: - uses: fwal/setup-swift@2040b795e5c453c3a05fcb8316496afc8a74f192 From ee36617cb8f093f0b5263ef7afce58633d0dc7a6 Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Fri, 6 May 2022 02:44:46 +0900 Subject: [PATCH 08/23] fixup --- .github/workflows/test.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 57a046a..b15827f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -21,5 +21,9 @@ jobs: - run: swift package resolve - name: Build run: swift build + - name: setup service + run: | + docker-compose -d db redis + sleep 10s - name: Run tests run: swift test From 8b95484c9d92ad501d7a55e4019cb39d1d99ddff Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Fri, 6 May 2022 02:52:48 +0900 Subject: [PATCH 09/23] fixup: nemuikamo --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b15827f..1f85b77 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -23,7 +23,7 @@ jobs: run: swift build - name: setup service run: | - docker-compose -d db redis + docker-compose up -d db redis sleep 10s - name: Run tests run: swift test From a1a464fac3c75e166867cea49d26f096d63dd825 Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Thu, 4 Jan 2024 22:34:09 +0900 Subject: [PATCH 10/23] =?UTF-8?q?chore:=20=E3=83=AD=E3=83=BC=E3=82=AB?= =?UTF-8?q?=E3=83=AB=E6=A4=9C=E8=A8=BC?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Sources/App/Commands/ScrapingAgqrCommand.swift | 1 + docker-compose.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/Sources/App/Commands/ScrapingAgqrCommand.swift b/Sources/App/Commands/ScrapingAgqrCommand.swift index 36ed7e5..b36fe99 100644 --- a/Sources/App/Commands/ScrapingAgqrCommand.swift +++ b/Sources/App/Commands/ScrapingAgqrCommand.swift @@ -34,6 +34,7 @@ struct ScrapingAgqr: Command { return } let programGuide = parser.parse(response) + context.console.info(programGuide.map { element in element.program.startDatetime.toString() }.joined(separator: "\n")) await repository.save(programGuide, app: context.application) } } diff --git a/docker-compose.yml b/docker-compose.yml index 03070fa..ddd5b50 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,6 +40,7 @@ services: command: ["serve", "--env", "production", "--hostname", "0.0.0.0", "--port", "8080"] dev: image: agqr-program-guide:latest + platform: linux/amd64 build: context: . target: develop @@ -49,6 +50,7 @@ services: - db commands: image: agqr-program-guide:latest + platform: linux/amd64 build: context: . environment: From e99b83dad64e8748db0fcf06e64c1a98d41acd05 Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Thu, 4 Jan 2024 23:48:13 +0900 Subject: [PATCH 11/23] =?UTF-8?q?fix:=20=E6=9A=AB=E5=AE=9A=E3=82=A8?= =?UTF-8?q?=E3=83=A9=E3=83=BC=E5=87=A6=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Sources/App/Commands/ScrapingAgqrCommand.swift | 12 ++++++++---- Sources/App/Jobs/ImportProgramGuideJob.swift | 3 +-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Sources/App/Commands/ScrapingAgqrCommand.swift b/Sources/App/Commands/ScrapingAgqrCommand.swift index b36fe99..3d4d4db 100644 --- a/Sources/App/Commands/ScrapingAgqrCommand.swift +++ b/Sources/App/Commands/ScrapingAgqrCommand.swift @@ -30,11 +30,15 @@ struct ScrapingAgqr: Command { func asyncRun(using context: CommandContext, signature: Signature) async { let response = await client.execute(app: context.application, url: signature.url) guard let response = response else { - print("htmlデータの取得に失敗しました") + context.console.error("htmlデータの取得に失敗しました") return } - let programGuide = parser.parse(response) - context.console.info(programGuide.map { element in element.program.startDatetime.toString() }.joined(separator: "\n")) - await repository.save(programGuide, app: context.application) + do { + let programGuide = try parser.parse(response) + context.console.info(programGuide.map { element in element.program.startDatetime.toString() }.joined(separator: "\n")) + await repository.save(programGuide, app: context.application) + } catch { + context.console.error(.init(stringLiteral: error.localizedDescription)) + } } } diff --git a/Sources/App/Jobs/ImportProgramGuideJob.swift b/Sources/App/Jobs/ImportProgramGuideJob.swift index 8c63232..857e36a 100644 --- a/Sources/App/Jobs/ImportProgramGuideJob.swift +++ b/Sources/App/Jobs/ImportProgramGuideJob.swift @@ -32,9 +32,8 @@ struct ImportProgramGuideJob: ScheduledJob { return await self.repository.save(programGuide, app: context.application) } catch let error as AgqrParseError { context.logger.error(.init(stringLiteral: error.message)) - return context.application.eventLoopGroup.future(error: error) } catch { - return context.application.eventLoopGroup.future(error: error) + context.logger.error(.init(stringLiteral: error.localizedDescription)) } } } From 02f33fc874a8dd2866456570c254b56e5aa24c3a Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Thu, 4 Jan 2024 23:48:47 +0900 Subject: [PATCH 12/23] =?UTF-8?q?feat:=20postgresql=20=E3=81=B8=E3=81=AE?= =?UTF-8?q?=E7=A7=BB=E8=A1=8C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .env.testing | 2 +- Package.resolved | 33 ++++++++++++++++++---------- Package.swift | 4 ++-- Sources/App/configure.swift | 15 ++++++------- docker-compose.yml | 13 +++++------ docker/db/01_create_databases.sql | 4 ++++ docker/mysql/01_create_databases.sql | 4 ---- 7 files changed, 41 insertions(+), 34 deletions(-) create mode 100755 docker/db/01_create_databases.sql delete mode 100755 docker/mysql/01_create_databases.sql diff --git a/.env.testing b/.env.testing index 4737a59..1d4e1d8 100644 --- a/.env.testing +++ b/.env.testing @@ -1,5 +1,5 @@ DATABASE_HOST=127.0.0.1 -DATABASE_PORT=3306 +DATABASE_PORT=5432 DATABASE_USERNAME=vapor_username DATABASE_PASSWORD=vapor_password DATABASE_NAME=test_agqr_program_guide diff --git a/Package.resolved b/Package.resolved index 4131c0e..69d8778 100644 --- a/Package.resolved +++ b/Package.resolved @@ -47,12 +47,12 @@ } }, { - "package": "fluent-mysql-driver", - "repositoryURL": "https://github.com/vapor/fluent-mysql-driver.git", + "package": "fluent-postgres-driver", + "repositoryURL": "https://github.com/vapor/fluent-postgres-driver.git", "state": { "branch": null, - "revision": "f86bf9c80a1c176234a16796c19add399a266c38", - "version": "4.0.2" + "revision": "7c266b539f71331ad6e53ea8fae587ccdaf972f2", + "version": "2.2.6" } }, { @@ -92,21 +92,21 @@ } }, { - "package": "mysql-kit", - "repositoryURL": "https://github.com/vapor/mysql-kit.git", + "package": "postgres-kit", + "repositoryURL": "https://github.com/vapor/postgres-kit.git", "state": { "branch": null, - "revision": "f54e0876fce2d68551be0e704b3a94f78eedb9f8", - "version": "4.5.0" + "revision": "35deea5c28a7d402f3280d81dee37bed5c56b9fe", + "version": "2.8.3" } }, { - "package": "mysql-nio", - "repositoryURL": "https://github.com/vapor/mysql-nio.git", + "package": "postgres-nio", + "repositoryURL": "https://github.com/vapor/postgres-nio.git", "state": { "branch": null, - "revision": "f0e8ad7e18e870e8665311d70bf3f01dcd6024a8", - "version": "1.4.0" + "revision": "d648c5b4594ffbc2f6173318f70f5531e05ccb4e", + "version": "1.11.0" } }, { @@ -163,6 +163,15 @@ "version": "3.18.0" } }, + { + "package": "swift-atomics", + "repositoryURL": "https://github.com/apple/swift-atomics.git", + "state": { + "branch": null, + "revision": "cd142fd2f64be2100422d658e7411e39489da985", + "version": "1.2.0" + } + }, { "package": "swift-backtrace", "repositoryURL": "https://github.com/swift-server/swift-backtrace.git", diff --git a/Package.swift b/Package.swift index e2be889..22097de 100644 --- a/Package.swift +++ b/Package.swift @@ -10,7 +10,7 @@ let package = Package( // 💧 A server-side Swift web framework. .package(url: "https://github.com/vapor/vapor.git", from: "4.0.0"), .package(url: "https://github.com/vapor/fluent.git", from: "4.0.0"), - .package(url: "https://github.com/vapor/fluent-mysql-driver.git", from: "4.0.0"), + .package(url: "https://github.com/vapor/fluent-postgres-driver.git", from: "2.0.0"), .package(url: "https://github.com/vapor/leaf.git", from: "4.0.0"), .package(url: "https://github.com/vapor/redis.git", from: "4.0.0"), .package(url: "https://github.com/vapor/queues-redis-driver.git", from: "1.0.0"), @@ -22,7 +22,7 @@ let package = Package( name: "App", dependencies: [ .product(name: "Fluent", package: "fluent"), - .product(name: "FluentMySQLDriver", package: "fluent-mysql-driver"), + .product(name: "FluentPostgresDriver", package: "fluent-postgres-driver"), .product(name: "Leaf", package: "leaf"), .product(name: "Redis", package: "redis"), .product(name: "QueuesRedisDriver", package: "queues-redis-driver"), diff --git a/Sources/App/configure.swift b/Sources/App/configure.swift index 73bd112..f332bd8 100644 --- a/Sources/App/configure.swift +++ b/Sources/App/configure.swift @@ -1,5 +1,5 @@ import Fluent -import FluentMySQLDriver +import FluentPostgresDriver import Leaf import Queues import QueuesRedisDriver @@ -9,17 +9,16 @@ import Vapor public func configure(_ app: Application) throws { // uncomment to serve files from /Public folder // app.middleware.use(FileMiddleware(publicDirectory: app.directory.publicDirectory)) - var tls = TLSConfiguration.makeClientConfiguration() - tls.certificateVerification = .none app.databases.use( - .mysql( + .postgres( hostname: Environment.get("DATABASE_HOST") ?? "localhost", - port: Environment.get("DATABASE_PORT").flatMap(Int.init(_:)) ?? MySQLConfiguration.ianaPortNumber, + port: Environment.get("DATABASE_PORT").flatMap(Int.init(_:)) ?? PostgresConfiguration.ianaPortNumber, username: Environment.get("DATABASE_USERNAME") ?? "vapor_username", password: Environment.get("DATABASE_PASSWORD") ?? "vapor_password", - database: Environment.get("DATABASE_NAME") ?? "vapor_database", - tlsConfiguration: tls - ), as: .mysql) + database: Environment.get("DATABASE_NAME") ?? "vapor_database" + ), + as: .psql + ) app.migrations.add(CreateProgram()) app.migrations.add(CreatePersonality()) diff --git a/docker-compose.yml b/docker-compose.yml index ddd5b50..2053c30 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -59,16 +59,15 @@ services: - db command: ["--help"] db: - image: mysql:8.0 + image: postgres:15.5 volumes: - - db_data:/var/lib/mysql - - ./docker/mysql:/docker-entrypoint-initdb.d + - db_data:/var/lib/postgresql + - ./docker/db:/docker-entrypoint-initdb.d environment: - MYSQL_USER: vapor_username - MYSQL_PASSWORD: vapor_password - MYSQL_RANDOM_ROOT_PASSWORD: 'yes' + POSTGRES_USER: vapor_username + POSTGRES_PASSWORD: vapor_password ports: - - '3306:3306' + - '5432:5432' redis: image: redis:latest ports: diff --git a/docker/db/01_create_databases.sql b/docker/db/01_create_databases.sql new file mode 100755 index 0000000..49f3ccc --- /dev/null +++ b/docker/db/01_create_databases.sql @@ -0,0 +1,4 @@ +CREATE DATABASE agqr_program_guide; +CREATE DATABASE test_agqr_program_guide; +GRANT ALL PRIVILEGES ON DATABASE agqr_program_guide TO vapor_username; +GRANT ALL PRIVILEGES ON DATABASE test_agqr_program_guide TO vapor_username; diff --git a/docker/mysql/01_create_databases.sql b/docker/mysql/01_create_databases.sql deleted file mode 100755 index 45a52b8..0000000 --- a/docker/mysql/01_create_databases.sql +++ /dev/null @@ -1,4 +0,0 @@ -CREATE DATABASE IF NOT EXISTS `agqr_program_guide`; -GRANT ALL ON `agqr_program_guide`.* TO 'vapor_username'@'%'; -CREATE DATABASE IF NOT EXISTS `test_agqr_program_guide`; -GRANT ALL ON `test_agqr_program_guide`.* TO 'vapor_username'@'%'; From 542c6e817fa649cac6cf58744ef937476dcfb29b Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Fri, 5 Jan 2024 00:15:51 +0900 Subject: [PATCH 13/23] =?UTF-8?q?feat:=20scheduled=20job=20=E3=82=92?= =?UTF-8?q?=E3=82=84=E3=82=81=E3=82=8B=E3=80=82Redis=20=E3=82=92=E6=B6=88?= =?UTF-8?q?=E3=81=99?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Package.resolved | 36 -------------- Package.swift | 4 -- .../App/Commands/ImportWeeklyPGCommand.swift | 49 +++++++++++++++++++ Sources/App/Jobs/ImportProgramGuideJob.swift | 40 --------------- Sources/App/configure.swift | 18 ++----- 5 files changed, 54 insertions(+), 93 deletions(-) create mode 100644 Sources/App/Commands/ImportWeeklyPGCommand.swift delete mode 100644 Sources/App/Jobs/ImportProgramGuideJob.swift diff --git a/Package.resolved b/Package.resolved index 69d8778..a4b0cf7 100644 --- a/Package.resolved +++ b/Package.resolved @@ -109,42 +109,6 @@ "version": "1.11.0" } }, - { - "package": "queues", - "repositoryURL": "https://github.com/vapor/queues.git", - "state": { - "branch": null, - "revision": "58b2d785118d164b38c59beb595cbffbea7608ef", - "version": "1.8.1" - } - }, - { - "package": "queues-redis-driver", - "repositoryURL": "https://github.com/vapor/queues-redis-driver.git", - "state": { - "branch": null, - "revision": "2728477b50e24be82f5bc0bd0722c35656e1c5b1", - "version": "1.0.3" - } - }, - { - "package": "redis", - "repositoryURL": "https://github.com/vapor/redis.git", - "state": { - "branch": null, - "revision": "e955843b08064071f465a6b1ca9e04bebad8623a", - "version": "4.6.0" - } - }, - { - "package": "RediStack", - "repositoryURL": "https://gitlab.com/mordil/RediStack.git", - "state": { - "branch": null, - "revision": "5458d6476e05d5f1b43097f1bc9b599e936b5f2f", - "version": "1.3.0" - } - }, { "package": "routing-kit", "repositoryURL": "https://github.com/vapor/routing-kit.git", diff --git a/Package.swift b/Package.swift index 22097de..b32d0a4 100644 --- a/Package.swift +++ b/Package.swift @@ -12,8 +12,6 @@ let package = Package( .package(url: "https://github.com/vapor/fluent.git", from: "4.0.0"), .package(url: "https://github.com/vapor/fluent-postgres-driver.git", from: "2.0.0"), .package(url: "https://github.com/vapor/leaf.git", from: "4.0.0"), - .package(url: "https://github.com/vapor/redis.git", from: "4.0.0"), - .package(url: "https://github.com/vapor/queues-redis-driver.git", from: "1.0.0"), // HTML Parser .package(url: "https://github.com/tid-kijyun/Kanna.git", .upToNextMajor(from: "5.2.4")) ], @@ -24,8 +22,6 @@ let package = Package( .product(name: "Fluent", package: "fluent"), .product(name: "FluentPostgresDriver", package: "fluent-postgres-driver"), .product(name: "Leaf", package: "leaf"), - .product(name: "Redis", package: "redis"), - .product(name: "QueuesRedisDriver", package: "queues-redis-driver"), .product(name: "Vapor", package: "vapor"), .product(name: "Kanna", package: "Kanna"), diff --git a/Sources/App/Commands/ImportWeeklyPGCommand.swift b/Sources/App/Commands/ImportWeeklyPGCommand.swift new file mode 100644 index 0000000..3e1a0db --- /dev/null +++ b/Sources/App/Commands/ImportWeeklyPGCommand.swift @@ -0,0 +1,49 @@ +import Fluent +import Vapor + +struct ImportWeeklyPGCommand: Command { + let parser: ProgramGuideParsing + let repository: ProgramGuideSaving + let client = DownloadAgqrProgramGuide() + + struct Signature: CommandSignature {} + + var help: String = "Import weekly program guides into db" + + func run(using context: CommandContext, signature: Signature) throws { + context.console.info("Start Process") + defer { + context.console.info("End Process") + } + + let promise = context.application.eventLoopGroup.next().makePromise(of: Void.self) + promise.completeWithTask { + await self.asyncRun(using: context, signature: signature) + } + + try promise.futureResult.wait() + } + + func asyncRun(using context: CommandContext, signature: Signature) async { + let responses = await client.fetchWeekly(app: context.application) + for response in responses { + guard let response = response else { + context.console.error("NotFound program data") + continue + } + do { + let programGuide = try self.parser.parse(response) + guard programGuide.count > 0 else { + context.console.error("parsed programs length is 0") + continue + } + await self.repository.save(programGuide, app: context.application) + context.console.info("success: \(programGuide[0].program.startDatetime)") + } catch let error as AgqrParseError { + context.console.error(.init(stringLiteral: error.message)) + } catch { + context.console.error(.init(stringLiteral: error.localizedDescription)) + } + } + } +} diff --git a/Sources/App/Jobs/ImportProgramGuideJob.swift b/Sources/App/Jobs/ImportProgramGuideJob.swift deleted file mode 100644 index 857e36a..0000000 --- a/Sources/App/Jobs/ImportProgramGuideJob.swift +++ /dev/null @@ -1,40 +0,0 @@ -import Foundation -import Queues -import Vapor - -struct ImportProgramGuideJob: ScheduledJob { - let parser: ProgramGuideParsing - let repository: ProgramGuideSaving - let client = DownloadAgqrProgramGuide() - - func run(context: QueueContext) -> EventLoopFuture { - // TODO: ログの生成場所・時間 - context.logger.info("Start Scraping Process") - defer { - context.logger.info("End Scraping Process") - } - let promise = context.eventLoop.makePromise(of: Void.self) - promise.completeWithTask { - await self.asyncRun(context: context) - } - return promise.futureResult - } - - func asyncRun(context: QueueContext) async { - let responses = await client.fetchWeekly(app: context.application) - for response in responses { - guard let response = response else { - context.logger.error("NotFound program data") - continue - } - do { - let programGuide = try self.parser.parse(response) - return await self.repository.save(programGuide, app: context.application) - } catch let error as AgqrParseError { - context.logger.error(.init(stringLiteral: error.message)) - } catch { - context.logger.error(.init(stringLiteral: error.localizedDescription)) - } - } - } -} diff --git a/Sources/App/configure.swift b/Sources/App/configure.swift index f332bd8..a7eee86 100644 --- a/Sources/App/configure.swift +++ b/Sources/App/configure.swift @@ -1,8 +1,6 @@ import Fluent import FluentPostgresDriver import Leaf -import Queues -import QueuesRedisDriver import Vapor // configures your application @@ -26,19 +24,13 @@ public func configure(_ app: Application) throws { app.views.use(.leaf) + let pgParser = DailyProgramGuideParser() + let pgRepository = ProgramGuideRepository() app.commands.use( - ScrapingAgqr(parser: DailyProgramGuideParser(), repository: ProgramGuideRepository()), as: "scraping") - - try app.queues.use(.redis(url: Environment.get("REDIS_URL") ?? "redis://127.0.0.1:6379")) - - app.queues.schedule( - ImportProgramGuideJob(parser: DailyProgramGuideParser(), repository: ProgramGuideRepository()) + ScrapingAgqr(parser: pgParser, repository: pgRepository), as: "scraping") + app.commands.use( + ImportWeeklyPGCommand(parser: pgParser, repository: pgRepository), as: "import:weekly" ) - .daily() - .at(7, 0) // 07:00 am - - // try app.queues.startInProcessJobs(on: .default) - try app.queues.startScheduledJobs() // register middleware let corsConfiguration = CORSMiddleware.Configuration( From b46afdcb5b9f6920ebd3e41e6610059cb436f8be Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Fri, 5 Jan 2024 00:31:47 +0900 Subject: [PATCH 14/23] =?UTF-8?q?chore:=20CI=E5=91=A8=E3=82=8A=E3=81=AE?= =?UTF-8?q?=E4=BF=AE=E6=AD=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build-push.yml | 8 ++++---- .github/workflows/test.yaml | 9 ++++----- Makefile | 2 +- docker-compose.yml => compose.yaml | 14 ++++++-------- 4 files changed, 15 insertions(+), 18 deletions(-) rename docker-compose.yml => compose.yaml (91%) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 29b25ea..1d63a2f 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -12,18 +12,18 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Set up Docker Buildx id: buildx - uses: docker/setup-buildx-action@v1.5.1 + uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry - uses: docker/login-action@v1 + uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push - uses: docker/build-push-action@v2 + uses: docker/build-push-action@v5 with: push: true tags: ghcr.io/${{ github.repository }}:${{ github.event.inputs.version }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1f85b77..b8310a5 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -11,19 +11,18 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] - swift: ["5.5"] + swift: ["5.9"] runs-on: ${{ matrix.os }} steps: - - uses: fwal/setup-swift@2040b795e5c453c3a05fcb8316496afc8a74f192 + - uses: swift-actions/setup-swift@v1 with: swift-version: ${{ matrix.swift }} - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: swift package resolve - name: Build run: swift build - name: setup service run: | - docker-compose up -d db redis - sleep 10s + docker compose up -d db --wait - name: Run tests run: swift test diff --git a/Makefile b/Makefile index 9156d94..bec2204 100644 --- a/Makefile +++ b/Makefile @@ -21,6 +21,6 @@ migrate: build: docker-compose build --no-cache up: - docker-compose up -d db redis + docker-compose up -d db down: docker-compose down diff --git a/docker-compose.yml b/compose.yaml similarity index 91% rename from docker-compose.yml rename to compose.yaml index 2053c30..6034180 100644 --- a/docker-compose.yml +++ b/compose.yaml @@ -14,8 +14,6 @@ # Run migrations: docker-compose run migrate # Stop all: docker-compose down (add -v to wipe db) # -version: '3.7' - volumes: db_data: @@ -68,9 +66,9 @@ services: POSTGRES_PASSWORD: vapor_password ports: - '5432:5432' - redis: - image: redis:latest - ports: - - '6379:6379' - volumes: - - ./data/redis:/data + healthcheck: + test: ["CMD", "pg_isready", "-U", "vapor_username"] + interval: 10s + timeout: 30s + retries: 5 + start_period: 30s From 8d2e8e2769134540308f5b9e873d5b01c714aa36 Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Fri, 5 Jan 2024 00:42:06 +0900 Subject: [PATCH 15/23] =?UTF-8?q?chore:=20macos=20=E3=81=9D=E3=81=AE?= =?UTF-8?q?=E3=81=BE=E3=81=BE=E3=81=A0=E3=81=A8=20docker=20=E4=BD=BF?= =?UTF-8?q?=E3=81=88=E3=81=AA=E3=81=84=E3=81=AE=E3=81=A7=E5=89=8A=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b8310a5..7e49897 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -10,7 +10,7 @@ jobs: name: Swift ${{ matrix.swift }} on ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, macos-latest] + os: [ubuntu-latest] swift: ["5.9"] runs-on: ${{ matrix.os }} steps: From a68fff2beda3d6d630259c694d237534d1462fb2 Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Fri, 5 Jan 2024 00:44:51 +0900 Subject: [PATCH 16/23] =?UTF-8?q?chore:=20run-name=20=E3=81=A7=E8=A6=8B?= =?UTF-8?q?=E3=82=84=E3=81=99=E3=81=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build-push.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/build-push.yml b/.github/workflows/build-push.yml index 1d63a2f..742d2f8 100644 --- a/.github/workflows/build-push.yml +++ b/.github/workflows/build-push.yml @@ -1,4 +1,5 @@ name: build-push +run-name: Push [${{ inputs.version }}] from ${{ github.ref_name }} on: workflow_dispatch: From 85ad1f11c9ff90164450039809283b04e23443d1 Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Mon, 8 Jan 2024 19:32:05 +0900 Subject: [PATCH 17/23] =?UTF-8?q?refactor:=20=E3=83=9F=E3=83=89=E3=83=AB?= =?UTF-8?q?=E3=82=A6=E3=82=A7=E3=82=A2=E7=94=9F=E6=88=90=E3=82=92=E9=96=A2?= =?UTF-8?q?=E6=95=B0=E3=81=AB=E5=88=87=E3=82=8A=E5=87=BA=E3=81=97?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Sources/App/configure.swift | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/Sources/App/configure.swift b/Sources/App/configure.swift index a7eee86..c2da4b1 100644 --- a/Sources/App/configure.swift +++ b/Sources/App/configure.swift @@ -33,6 +33,13 @@ public func configure(_ app: Application) throws { ) // register middleware + app.middleware.use(newCORSMiddleware()) + + // register routes + try routes(app) +} + +func newCORSMiddleware() -> CORSMiddleware { let corsConfiguration = CORSMiddleware.Configuration( allowedOrigin: .all, allowedMethods: [.GET, .OPTIONS, .HEAD], @@ -40,8 +47,5 @@ public func configure(_ app: Application) throws { .accept, .contentType, .origin, .xRequestedWith, .userAgent, .accessControlAllowOrigin, ] ) - app.middleware.use(CORSMiddleware(configuration: corsConfiguration)) - - // register routes - try routes(app) + return CORSMiddleware(configuration: corsConfiguration) } From 76f629b69d69c275eddfee6daecd72f7a9fe1f72 Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Mon, 8 Jan 2024 19:52:45 +0900 Subject: [PATCH 18/23] =?UTF-8?q?chore:=20openapi=20=E3=81=AE=E8=AA=BF?= =?UTF-8?q?=E6=95=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../agqr-radio-program-guide-api.v2.yaml | 132 +++--------------- 1 file changed, 20 insertions(+), 112 deletions(-) diff --git a/reference/agqr-radio-program-guide-api.v2.yaml b/reference/agqr-radio-program-guide-api.v2.yaml index a1ca023..6a236ba 100644 --- a/reference/agqr-radio-program-guide-api.v2.yaml +++ b/reference/agqr-radio-program-guide-api.v2.yaml @@ -2,88 +2,26 @@ openapi: 3.0.0 info: title: agqr-radio-program-guide-api version: '1.0' - contact: - name: sun-yryr(t_minagawa) - url: 'https://twitter.com/taittide' - email: taittide@gmail.com + description: |- + **[非公式] 超A&G+ 番組表 API** + データ更新頻度: 1日ごと（JST 07:00 ごろ） + + 何かバグ等ありましたら、Issue を立てていただくか下記の連絡先までご連絡ください。 + Twitter(X): [@sun_yryr](https://twitter.com/sun_yryr) + GitHub: https://github.com/sun-yryr/agqr-program-guide servers: + - url: 'https://agqr.sun-yryr.com/api' + description: production - url: 'http://localhost:8080/api' description: local - - url: 'https://agqr.sun-yryr.com/api' - description: prod +tags: + - name: v1 + description: 'v1 API' paths: - /v2/programs/weekly: - get: - summary: '[未実装] 週間番組表を取得' - tags: [] - responses: - '200': - description: OK（検索結果がない場合は空配列になります） - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/program' - operationId: get-v2-programs-weekly - description: 条件に合った番組情報を返却する。 - parameters: - - schema: - type: string - in: query - name: q - description: '検索ワード（title,personality,infoの全てを対象）' - - schema: - type: string - example: 'audio,movie,repeat' - default: 'audio,movie' - in: query - name: include_program_type - description: |- - 検索結果に含める番組タイプ - [,]区切りで記述 - movie: 動画付きのみ - audio: 音声のみ - repeat: 再放送を含める - parameters: [] - /v2/programs/daily: - get: - summary: '[未実装] 日間番組表を取得' - tags: [] - responses: - '200': - description: OK（検索結果がない場合は空配列になります） - content: - application/json: - schema: - type: array - items: - $ref: '#/components/schemas/program' - operationId: get-v2-programs-daily - description: 条件に合った番組情報を返却する - parameters: - - schema: - type: string - in: query - name: q - description: '検索ワード（title,personality,infoの全てを対象）' - - schema: - type: string - example: 'audio,movie,repeat' - default: 'audio,movie' - in: query - name: include_program_type - description: |- - 検索結果に含める番組タイプ - [,]区切りで記述 - movie: 動画付きのみ - audio: 音声のみ - repeat: 再放送を含める - parameters: [] /all: get: summary: 週刊番組表を取得 - tags: [] + tags: [v1] responses: '200': description: OK @@ -95,12 +33,12 @@ paths: $ref: '#/components/schemas/old-program' operationId: get-all description: |- - 1週間分全ての番組情報を返却します - そのうち削除予定 + 1週間分全ての番組情報を返却する。 + 実行日の日本時間の0時から7日後の23時59分までの間の番組情報を返却する。 /today: get: summary: 日間番組表を取得 - tags: [] + tags: [v1] responses: '200': description: OK @@ -112,12 +50,12 @@ paths: $ref: '#/components/schemas/old-program' operationId: get-today description: |- - 本日放送予定の番組情報を返却します - そのうち削除予定 + 本日放送予定の番組情報を返却する。 + 本日とは、日本時間の0時から23時59分までの間のことを指す。 /now: get: summary: 現在の番組情報を返却 - tags: [] + tags: [v1] responses: '200': description: OK @@ -128,37 +66,7 @@ paths: items: $ref: '#/components/schemas/old-program' operationId: get-now - description: |- - 現在放送中の番組情報を返却します - そのうち削除予定 - /v2/programs/now: - get: - summary: '[未実装] 現在の番組情報を返却' - tags: [] - responses: - '200': - description: OK - content: - application/json: - schema: - $ref: '#/components/schemas/program' - '404': - description: 現在放送中の番組がありません - operationId: get-v2-programs-now - description: 現在放送中の番組情報を返却します - '/v2/programs/{programId}': - parameters: - - schema: - type: string - name: programId - in: path - required: true - get: - summary: '[未実装] 番組詳細を取得する' - tags: [] - responses: {} - operationId: get-v2-programs-programId - description: '' + description: 現在放送中の番組情報を返却する components: schemas: program: From e0bc7259f2f49087269330a564455de39a798b98 Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Mon, 8 Jan 2024 19:54:30 +0900 Subject: [PATCH 19/23] =?UTF-8?q?fix:=20=E3=83=98=E3=83=AB=E3=82=B9?= =?UTF-8?q?=E3=83=81=E3=82=A7=E3=83=83=E3=82=AF=E3=82=92=E6=AD=A3=E3=81=97?= =?UTF-8?q?=E3=81=8F=E7=A4=BA=E3=81=99=20URL=20=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Sources/App/routes.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Sources/App/routes.swift b/Sources/App/routes.swift index f76ac49..4cf96ac 100644 --- a/Sources/App/routes.swift +++ b/Sources/App/routes.swift @@ -6,8 +6,8 @@ func routes(_ app: Application) throws { return req.view.render("index", ["title": "Hello Vapor!"]) } - app.get("hello") { _ -> String in - return "Hello, world!" + app.get("health") { _ -> Response in + return Response(status: .ok) } try app.group("api") { builder in From c719f08705049cec5c0e6593ea8b6d55b246c47e Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Mon, 8 Jan 2024 20:21:28 +0900 Subject: [PATCH 20/23] =?UTF-8?q?chore:=20redoc=20=E3=82=92=E7=94=9F?= =?UTF-8?q?=E6=88=90=E3=81=97=E3=81=A6=E3=83=AB=E3=83=BC=E3=83=88=E3=81=AB?= =?UTF-8?q?=E5=9F=8B=E3=82=81=E3=82=8B?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Makefile | 5 + Resources/Views/redoc-static.html | 338 ++++++++++++++++++++++++++++++ Sources/App/routes.swift | 2 +- 3 files changed, 344 insertions(+), 1 deletion(-) create mode 100644 Resources/Views/redoc-static.html diff --git a/Makefile b/Makefile index bec2204..bc99607 100644 --- a/Makefile +++ b/Makefile @@ -24,3 +24,8 @@ up: docker-compose up -d db down: docker-compose down + +.PHONY: generate-redoc +generate-redoc: + docker run --rm -v ${PWD}/reference:/spec redocly/cli build-docs agqr-radio-program-guide-api.v2.yaml -o redoc-static.html + mv reference/redoc-static.html Resources/Views/redoc-static.html diff --git a/Resources/Views/redoc-static.html b/Resources/Views/redoc-static.html new file mode 100644 index 0000000..2b56195 --- /dev/null +++ b/Resources/Views/redoc-static.html @@ -0,0 +1,338 @@ + + + + + + agqr-radio-program-guide-api + + + + + + + + + +
API docs by Redocly

agqr-radio-program-guide-api (1.0)

Download OpenAPI specification:Download

[非公式] 超A&G+ 番組表 API
データ更新頻度: 1日ごと（JST 07:00 ごろ）

+

何かバグ等ありましたら、Issue を立てていただくか下記の連絡先までご連絡ください。
Twitter(X): @sun_yryr
GitHub: https://github.com/sun-yryr/agqr-program-guide

+

v1

v1 API

+

週刊番組表を取得

1週間分全ての番組情報を返却する。 +実行日の日本時間の0時から7日後の23時59分までの間の番組情報を返却する。

+

Responses

Response samples

Content type
application/json
[
  • {
    }
]

日間番組表を取得

本日放送予定の番組情報を返却する。 +本日とは、日本時間の0時から23時59分までの間のことを指す。

+

Responses

Response samples

Content type
application/json
[
  • {
    }
]

現在の番組情報を返却

現在放送中の番組情報を返却する

+

Responses

Response samples

Content type
application/json
[
  • {
    }
]
+ + + + diff --git a/Sources/App/routes.swift b/Sources/App/routes.swift index 4cf96ac..c265835 100644 --- a/Sources/App/routes.swift +++ b/Sources/App/routes.swift @@ -3,7 +3,7 @@ import Vapor func routes(_ app: Application) throws { app.get { req in - return req.view.render("index", ["title": "Hello Vapor!"]) + return req.view.render("redoc-static.html") } app.get("health") { _ -> Response in From 2dc8da253c571a6d35573b125c5c1cc935a91a61 Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Mon, 8 Jan 2024 20:32:10 +0900 Subject: [PATCH 21/23] =?UTF-8?q?chore:=20openapi=E3=82=92=E3=83=81?= =?UTF-8?q?=E3=82=A7=E3=83=83=E3=82=AF=E3=81=99=E3=82=8BCI=E3=81=AE?= =?UTF-8?q?=E8=BF=BD=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/check-openapi.yaml | 34 ++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 .github/workflows/check-openapi.yaml diff --git a/.github/workflows/check-openapi.yaml b/.github/workflows/check-openapi.yaml new file mode 100644 index 0000000..8f56b0b --- /dev/null +++ b/.github/workflows/check-openapi.yaml @@ -0,0 +1,34 @@ +name: Check OpenAPI +on: + push: + branches: + - master + paths: + - '.github/workflows/check-openapi.yaml' + - 'reference/agqr-radio-program-guide-api.v2.yaml' + - 'Resources/Views/redoc-static.html' + pull_request: + branches: + - master + paths: + - '.github/workflows/check-openapi.yaml' + - 'reference/agqr-radio-program-guide-api.v2.yaml' + - 'Resources/Views/redoc-static.html' + +jobs: + exists-diff: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - run: make generate-redoc + - name: Check exists diff + run: | + if ! git diff --quiet; then + exit 1 + fi + # TODO: 後で直す + # lint: + # runs-on: ubuntu-latest + # steps: + # - uses: actions/checkout@v4 + # - run: make lint-openapi From d53f06bc842ec1731038f9cf13d7689dfaffe75d Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Mon, 8 Jan 2024 20:41:48 +0900 Subject: [PATCH 22/23] =?UTF-8?q?test:=20hello=20=E3=82=A8=E3=83=B3?= =?UTF-8?q?=E3=83=89=E3=83=9D=E3=82=A4=E3=83=B3=E3=83=88=E3=81=AB=E5=AF=BE?= =?UTF-8?q?=E3=81=97=E3=81=A6=E3=81=AE=E3=83=86=E3=82=B9=E3=83=88=E3=82=92?= =?UTF-8?q?=20health=20=E3=82=A8=E3=83=B3=E3=83=89=E3=83=9D=E3=82=A4?= =?UTF-8?q?=E3=83=B3=E3=83=88=E3=81=AB=E5=A4=89=E6=9B=B4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Tests/AppTests/AppTests.swift | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Tests/AppTests/AppTests.swift b/Tests/AppTests/AppTests.swift index 1d4a8d0..11597a8 100644 --- a/Tests/AppTests/AppTests.swift +++ b/Tests/AppTests/AppTests.swift @@ -2,10 +2,9 @@ import XCTVapor final class AppTests: ControllerBaseTestCase { - func testHelloWorld() throws { - try app.test(.GET, "hello", afterResponse: { res in + func testHealth() throws { + try app.test(.GET, "health", afterResponse: { res in XCTAssertEqual(res.status, .ok) - XCTAssertEqual(res.body.string, "Hello, world!") }) } } From 9483f81624d4ad81507e25fbd4baa1c3e34df0dc Mon Sep 17 00:00:00 2001 From: sun-yryr Date: Mon, 8 Jan 2024 21:12:39 +0900 Subject: [PATCH 23/23] chore: update README --- README.md | 59 ++----------------------------------------------------- 1 file changed, 2 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index b42649d..6d33fa8 100644 --- a/README.md +++ b/README.md @@ -1,62 +1,7 @@ # agqr-program-guide -[超A&G+の番組表](https://www.joqr.co.jp/qr/agdailyprogram/) から番組情報・出演者を取得し、JSON形式のREST APIとして提供しています。 +[超A&G+の番組表](https://www.joqr.co.jp/qr/agdailyprogram/) から番組情報・出演者を取得し、JSON 形式の REST API として提供しています。 https://agqr.sun-yryr.com で提供中です。 -フレームワークとしてSwift製のVaporを利用しています。 - -## Development - -前提 -```bash -swift --version -# 5.4.2 -vapor --version -# framework: 4.45.0 -# toolbox: 18.3.3 -``` - -1. DBとRedisを起動する。 `make up` -1. 開発する。 -1. サーバーを立てて動作確認する。 `make serve` - -### lint - -swiftlintかswift-formatを利用する予定。更新する。 - -### build & push - -GitHub Actionsのページから `build-push` のワークフローを実行する。 - -### deploy - -1. EC2にログインする。 -1. pushしたDockerImageをPullしてくる。（直接起動する場合は飛ばしてもいい） -1. 環境変数とかを設定して起動する。下記の「使う」を参考にする。 - -## 使う - -適宜タグや環境変数の変更が必要。 - -### マイグレーション -```bash -docker run --rm -e DATABASE_HOST=127.0.0.1 -e REDIS_URL="redis://127.0.0.1:6379" ghcr.io/sun-yryr/agqr-program-guide:latest migrate --yes -``` - -### 手動スクレイピング - -基本的に自動で取ってくるので必要ない。初回起動などで利用する。 -※ hoge は使っていない引数なので後々削除する。 -```bash -docker run --rm -e DATABASE_HOST=127.0.0.1 -e REDIS_URL="redis://127.0.0.1:6379" -e TZ=Asia/Tokyo ghcr.io/sun-yryr/agqr-program-guide:latest scraping hoge -``` - -### 起動 -```bash -docker run -d -e DATABASE_HOST=127.0.0.1 -e REDIS_URL="redis://127.0.0.1:6379" -e TZ=Asia/Tokyo -p 3000:8080 ghcr.io/sun-yryr/agqr-program-guide:latest -``` - -## ライセンス - -そのうち +フレームワークとして [Vapor](https://github.com/vapor/vapor) を利用しています。