## Set the GitLab Token, check that it's OK
## export GITLAB_TOKEN=...
. $HOME/gitlab-token.sh
glab auth status

## Set the GitLab User and Repo for posting GitLab Snippets
export GITLAB_USER=lupyuen
export GITLAB_REPO=nuttx-build-log

## Download the NuttX Rewind Script
git clone https://github.com/lupyuen/nuttx-build-farm
cd nuttx-build-farm

## Find the Breaking Commit for QEMU RISC-V (64-bit Kernel Build)
nuttx_hash=  ## Optional: Begin with this NuttX Hash
apps_hash=   ## Optional: Begin with this Apps Hash
./rewind-build.sh \
  rv-virt:knsh64_test \
  $nuttx_hash \
  $apps_hash

## Build and Test: Latest NuttX Commit
git reset --hard HEAD
tools/configure.sh rv-virt:knsh64
make -j
qemu-system-riscv64 -kernel nuttx

## Build and Test: Previous NuttX Commit
git reset --hard HEAD~1
tools/configure.sh rv-virt:knsh64
make -j
qemu-system-riscv64 -kernel nuttx
...
## Build and Test: 20th NuttX Commit
git reset --hard HEAD~19
tools/configure.sh rv-virt:knsh64
make -j
qemu-system-riscv64 -kernel nuttx

## Roughly One Hour for 20 Rewinds of Build + Test

## Build and Test NuttX for QEMU RISC-V 64-bit (Kernel Build)
## Download NuttX and Apps
git clone https://github.com/apache/nuttx
git clone https://github.com/apache/nuttx-apps apps

## Switch to this NuttX Commit and Apps Commit
pushd nuttx ; git reset --hard $nuttx_hash ; popd
pushd apps  ; git reset --hard $apps_hash  ; popd

## Configure the NuttX Build
cd nuttx
tools/configure.sh rv-virt:knsh64

## Build the NuttX Kernel
make -j

## Build the NuttX Apps
make -j export
pushd ../apps
./tools/mkimport.sh -z -x ../nuttx/nuttx-export-*.tar.gz
make -j import
popd

## Boot NuttX on QEMU RISC-V 64-bit
## Run OSTest with our Expect Script
wget https://raw.githubusercontent.com/lupyuen/nuttx-riscv64/main/qemu-riscv-knsh64.exp
expect qemu-riscv-knsh64.exp

## Build and Test One Commit
function build_nuttx { ...

  ## NuttX Dashboard expects this Log Format
  echo "===================================================================================="
  echo "Configuration/Tool: rv-virt/knsh64_test,"
  echo "$timestamp"
  echo "------------------------------------------------------------------------------------"

  ## Build and Test Locally: QEMU RISC-V 64-bit Kernel Build
  $script_dir/build-test-knsh64.sh \
    $nuttx_commit \
    $apps_commit
  res=$?

  ## Omitted: Build and Test Other Targets
  echo "===================================================================================="
}

## Build and Test This Commit
build_nuttx $nuttx_hash $apps_hash

## If Build / Test Fails...
if [[ "$res" != "0" ]]; then
  echo "BUILD / TEST FAILED FOR THIS COMMIT: nuttx @ $nuttx_hash / nuttx-apps @ $apps_hash"

  ## Rebuild / Retest with the Previous Commit
  build_nuttx $prev_hash $apps_hash
  if [[ "$res" != "0" ]]; then
    echo "BUILD / TEST FAILED FOR PREVIOUS COMMIT: nuttx @ $prev_hash / nuttx-apps @ $apps_hash"
  fi

  ## Rebuild / Retest with the Next Commit
  build_nuttx $next_hash $apps_hash
  if [[ "$res" != "0" ]]; then
    echo "BUILD / TEST FAILED FOR NEXT COMMIT: nuttx @ $next_hash / nuttx-apps @ $apps_hash"
  fi
fi

## Why the Long Echoes? We'll ingest them later

## Build and Test This Commit
## And capture the Build-Test Output
function run_job { ...
  script $log_file \
    $script_option \
    " \
      $script_dir/rewind-commit.sh \
        $target \
        $nuttx_hash $apps_hash \
        $timestamp \
        $prev_hash $next_hash \
    "
}

## Build and Test One Commit for the NuttX Target
function build_commit { ...

  ## Build and Test This Commit
  ## And capture the Build-Test Output into a Log File
  run_job \
    $log $timestamp \
    $apps_hash $nuttx_hash \
    $prev_hash $next_hash
  clean_log $log
  find_messages $log

  ## Upload the Build-Test Log File
  ## As GitLab Snippet
  upload_log \
    $log unknown \
    $nuttx_hash $apps_hash \
    $timestamp
}

## Build and Test the Latest 20 Commits
num_commits=20
num_success=0
count=1
for commit in $(
  TZ=UTC0 \
  git log \
  -$(( $num_commits + 1 )) \
  --date='format-local:%Y-%m-%dT%H:%M:%S' \
  --format="%cd,%H"
); do
  ## Extract the Commit Timestamp and Commit Hash
  ## Commit looks like 2024-11-24T09:52:42,9f9cc7ecebd97c1a6b511a1863b1528295f68cd7
  prev_timestamp=$(echo $commit | cut -d ',' -f 1)  ## 2024-11-24T09:52:42
  prev_hash=$(echo $commit | cut -d ',' -f 2)       ## 9f9cc7ecebd97c1a6b511a1863b1528295f68cd7

  ## Build and Test the NuttX Hash + Apps Hash
  ## If It Fails: Build and Test the Previous NuttX Hash + Previous Apps Hash
  build_commit \
    $tmp_dir/$nuttx_hash.log \
    $timestamp \
    $apps_hash $nuttx_hash \
    $prev_hash $next_hash

  ## Shift the Commits
  ## Omitted: Skip the First Commit (because we need a Previous Commit)
  next_hash=$nuttx_hash
  nuttx_hash=$prev_hash
  timestamp=$prev_timestamp
  ((count++)) || true

  ## Stop when we have reached the
  ## Minimum Number of Successful Commits
  if [[ "$num_success" == "$min_commits" ]]; then
    break
  fi
done

# TYPE build_score gauge
# HELP build_score 1.0 for successful build, 0.0 for failed build
build_score{ 

  ## These fields shall be rendered in Grafana (NuttX Dashboard and Build History)
  timestamp="2025-01-11T10:54:36",
  user="rewind",
  board="rv-virt",
  config="knsh64_test",
  target="rv-virt:knsh64_test",
  url="https://gitlab.com/lupyuen/nuttx-build-log/-/snippets/4800059#L85",

  ## Here's the NuttX Hash and Apps Hash for This Commit
  nuttx_hash="657247bda89d60112d79bb9b8d223eca5f9641b5",
  apps_hash="a6b9e718460a56722205c2a84a9b07b94ca664aa",

  ## Previous Commit is OK (Score=1)
  nuttx_hash_prev="be40c01ddd6f43a527abeae31042ba7978aabb58",
  apps_hash_prev="a6b9e718460a56722205c2a84a9b07b94ca664aa",
  build_score_prev="1",

  ## Next Commit is Not OK (Score=0)
  nuttx_hash_next="48846954d8506e1c95089a8654787fdc42cc098c",
  apps_hash_next="a6b9e718460a56722205c2a84a9b07b94ca664aa",
  build_score_next="0"

} 0  ## Means This Commit Failed (Score=0)

// Post the Test Log to Prometheus Pushgateway:
// Compose the Pushgateway Metric containing the Test Log
let body = format!(r##"
build_score ... url="{url}" ... {build_score}
"##);

// Send the Metric to Pushgateway via HTTP POST
let client = reqwest::Client::new();
let pushgateway = format!("http://localhost:9091/metrics/job/{user}/instance/{target_rewind}");
let res = client
  .post(pushgateway)
  .body(body)
  .send()
  .await?;

build_score{
  target="rv-virt:knsh64_test",
  build_score_prev="1"
} == 0

// Query Prometheus for the Breaking Commit
let query = format!(r##"
  build_score{ ...
    target="{TARGET}",
    build_score_prev="1"
  ... } == 0
"##);

// Send query to Prometheus via HTTP Form Post
let params = [("query", query)];
let client = reqwest::Client::new();
let prometheus = format!("http://{prometheus_server}/api/v1/query");
let res = client
  .post(prometheus)
  .form(&params)
  .send()
  .await?;

// Process the Query Results (Breaking Commit)
let body = res.text().await?;
let data: Value = serde_json::from_str(&body).unwrap();
let builds = &data["data"]["result"];

## Fetch the Pull Request for this Commit
$ commit=be40c01ddd6f43a527abeae31042ba7978aabb58
$ curl -L \
  -H "Accept: application/vnd.github+json" \
  -H "X-GitHub-Api-Version: 2022-11-28" \
  https://api.github.com/repos/apache/nuttx/commits/$commit/pulls

[{ html_url: "https://github.com/apache/nuttx/pull/15444",
   title:    "modlib: preprocess gnu-elf.ld for executable ELF",
   user: { login: "GITHUB_USERID", ...

## Add a Cron Job
$ crontab -e

## Then insert this...
## Test and Rewind: Every Day at 00:00 UTC
0 0 * * * /home/luppy/nuttx-build-farm/cron.sh 2>&1 | logger -t nuttx-rewind-build

## Or For Testing...
## Test and Rewind: Every Hour at 00:16, 01:16, 12:16, ...
16 * * * * /home/luppy/nuttx-build-farm/cron.sh 2>&1 | logger -t nuttx-rewind-build

## Exit and Monitor our Cron Job
$ tail -f /var/log/syslog

## Add a Cron Job
$ crontab -e

## Then insert this...
## Notify Mastodon: Every 15 minutes
*/15 * * * * /home/luppy/nuttx-rewind-notify/cron.sh 2>&1 | logger -t nuttx-rewind-notify

## Or For Testing...
## Notify Mastodon: Every Hour at 00:16, 01:16, 12:16, ...
16 * * * * /home/luppy/nuttx-rewind-notify/cron.sh 2>&1 | logger -t nuttx-rewind-notify

## Exit and Monitor our Cron Job
$ tail -f /var/log/syslog