Janis Eccarius
1c80abdb8a
Build & Test (NowChessSystems) TeamCity build finished
Add an easy local data pipeline feeding GPU training on Colab. - SelfPlayMain: standalone NNUEBot self-play (no microservices) writing FENs for labeling; randomised openings for game diversity, sequential due to the shared EvaluationNNUE accumulator. Exposed via the `selfPlay` Gradle task and selfplay.sh. - NNUEBot: optional fixedMoveTimeMs so self-play runs fast (default unchanged). - NbaiLoader: honor `-Dnnue.weights=<path>` to load weights from a file before falling back to the bundled resource. - build_dataset.py / dataset.sh: one command builds the entire dataset (Lichess eval-DB backbone + self-play + tactical + random filler), dedups, balances the eval histogram, writes append-only zstd shards + manifest, and rclone-pushes to Drive. - train.py: NNUEDataset reads a directory of .jsonl.zst shards (streaming) in addition to a single file. - NNUETraining.ipynb: clone to ephemeral /content, sync shards from Drive (cache-aware), train on the shards dir; removed Colab generation/upload steps. - Concept + implementation plan docs. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
133 lines
4.4 KiB
Kotlin
133 lines
4.4 KiB
Kotlin
plugins {
|
|
id("scala")
|
|
id("org.scoverage") version "8.1"
|
|
id("io.quarkus")
|
|
}
|
|
|
|
group = "de.nowchess"
|
|
version = "1.0-SNAPSHOT"
|
|
|
|
@Suppress("UNCHECKED_CAST")
|
|
val versions = rootProject.extra["VERSIONS"] as Map<String, String>
|
|
@Suppress("UNCHECKED_CAST")
|
|
val scoverageExcluded = rootProject.extra["SCOVERAGE_EXCLUDED"] as List<String>
|
|
|
|
repositories {
|
|
mavenCentral()
|
|
}
|
|
|
|
scala {
|
|
scalaVersion = versions["SCALA3"]!!
|
|
}
|
|
|
|
scoverage {
|
|
scoverageVersion.set(versions["SCOVERAGE"]!!)
|
|
excludedPackages.set(
|
|
listOf(
|
|
"de\\.nowchess\\.bot\\.bots\\.NNUEBot",
|
|
"de\\.nowchess\\.bot\\.bots\\.nnue\\..*",
|
|
"de\\.nowchess\\.bot\\.util\\.PolyglotBook",
|
|
"de\\.nowchess\\.bot\\.resource\\..*",
|
|
"de\\.nowchess\\.bot\\.config\\..*",
|
|
)
|
|
)
|
|
excludedFiles.set(scoverageExcluded)
|
|
}
|
|
|
|
val quarkusPlatformGroupId: String by project
|
|
val quarkusPlatformArtifactId: String by project
|
|
val quarkusPlatformVersion: String by project
|
|
|
|
tasks.withType<ScalaCompile> {
|
|
scalaCompileOptions.additionalParameters = listOf("-encoding", "UTF-8")
|
|
}
|
|
|
|
tasks.withType<JavaCompile> {
|
|
options.encoding = "UTF-8"
|
|
options.compilerArgs.add("-parameters")
|
|
}
|
|
|
|
tasks.register<JavaExec>("selfPlay") {
|
|
group = "nnue"
|
|
description = "Run standalone NNUEBot self-play and write FENs for labeling."
|
|
mainClass.set("de.nowchess.bot.selfplay.SelfPlayMain")
|
|
classpath = sourceSets["main"].runtimeClasspath
|
|
args((project.findProperty("spArgs")?.toString() ?: "").split(" ").filter { it.isNotBlank() })
|
|
}
|
|
|
|
dependencies {
|
|
|
|
compileOnly("org.scala-lang:scala3-compiler_3") {
|
|
version {
|
|
strictly(versions["SCALA3"]!!)
|
|
}
|
|
}
|
|
implementation("org.scala-lang:scala3-library_3") {
|
|
version {
|
|
strictly(versions["SCALA3"]!!)
|
|
}
|
|
}
|
|
|
|
implementation(platform("${quarkusPlatformGroupId}:${quarkusPlatformArtifactId}:${quarkusPlatformVersion}"))
|
|
implementation("io.quarkus:quarkus-rest")
|
|
implementation("io.quarkus:quarkus-rest-jackson")
|
|
implementation("io.quarkus:quarkus-arc")
|
|
implementation("io.quarkus:quarkus-config-yaml")
|
|
implementation("io.quarkus:quarkus-smallrye-jwt")
|
|
implementation("io.quarkus:quarkus-smallrye-health")
|
|
implementation("io.quarkus:quarkus-smallrye-openapi")
|
|
implementation("io.quarkus:quarkus-logging-json")
|
|
implementation("io.quarkus:quarkus-micrometer")
|
|
implementation("io.quarkus:quarkus-micrometer-registry-prometheus")
|
|
implementation("io.quarkus:quarkus-opentelemetry")
|
|
implementation("com.fasterxml.jackson.module:jackson-module-scala_3:${versions["JACKSON_SCALA"]!!}")
|
|
|
|
implementation(project(":modules:api"))
|
|
implementation(project(":modules:io"))
|
|
implementation(project(":modules:rule"))
|
|
implementation(project(":modules:security"))
|
|
implementation("io.quarkus:quarkus-rest-client-jackson")
|
|
implementation("com.microsoft.onnxruntime:onnxruntime:${versions["ONNXRUNTIME"]!!}")
|
|
implementation("io.quarkus:quarkus-redis-client")
|
|
|
|
testImplementation(platform("org.junit:junit-bom:${versions["JUNIT_BOM"]!!}"))
|
|
testImplementation("org.junit.jupiter:junit-jupiter")
|
|
testImplementation("org.scalatest:scalatest_3:${versions["SCALATEST"]!!}")
|
|
testImplementation("co.helmethair:scalatest-junit-runner:${versions["SCALATEST_JUNIT"]!!}")
|
|
testImplementation("io.quarkus:quarkus-junit")
|
|
testImplementation("io.rest-assured:rest-assured")
|
|
testImplementation("io.quarkus:quarkus-test-security")
|
|
|
|
testRuntimeOnly("org.junit.platform:junit-platform-launcher")
|
|
}
|
|
|
|
configurations.matching { !it.name.startsWith("scoverage") }.configureEach {
|
|
resolutionStrategy.force("org.scala-lang:scala-library:${versions["SCALA_LIBRARY"]!!}")
|
|
}
|
|
configurations.scoverage {
|
|
resolutionStrategy.eachDependency {
|
|
if (requested.group == "org.scoverage" && requested.name.startsWith("scalac-scoverage-plugin_")) {
|
|
useTarget("${requested.group}:scalac-scoverage-plugin_2.13.16:2.3.0")
|
|
}
|
|
}
|
|
}
|
|
|
|
tasks.withType<Jar>().configureEach {
|
|
duplicatesStrategy = DuplicatesStrategy.EXCLUDE
|
|
}
|
|
|
|
tasks.test {
|
|
useJUnitPlatform {
|
|
includeEngines("scalatest", "junit-jupiter")
|
|
testLogging {
|
|
events("passed", "skipped", "failed")
|
|
showStandardStreams = true
|
|
exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL
|
|
}
|
|
}
|
|
finalizedBy(tasks.reportScoverage)
|
|
}
|
|
tasks.reportScoverage {
|
|
dependsOn(tasks.test)
|
|
}
|