add release script (#636)

add release script to start automating our releases with `cargo-release`
and `git-cliff`. the main release script is `./release.sh` with rather
straightforward usage. we use `cargo-release` to handle some of the
`Cargo.toml` updates and trigger git-cliff, but otherwise the script
just manually uses cargo/git to publish the release and tag it

the release flow will be the following:
1. from an up-to-date `main` branch, checkout a new release branch (like
`prepare 0.x.y`)
2. run `./release 0.x.y` to do the 'prepare' phase of generating a
changelog, updating readme etc.
a. the tool will validate that you're on a release branch, generate the
changelog, and make a commit and (optionally) open a PR (cargo-release
is used to update the versions correctly and git-cliff is used as a hook
in cargo-release to generate the changelog)
b. then the only main action is just making minor CHANGELOG changes and
updating + merging the PR
3. merge the 'prepare' PR
4. from `main` branch with prepare branch merged, run: `./release.sh`
a. the tool will publish any unpublished version (e.g. if 0.6.0 is
latest on crates.io and the prepare PR bumps to 0.6.1 then the tool will
do the release for `delta_kernel` and `delta_kernel_derive`, add a git
tag, and push the tag)
b. NOTE: this is currently done manually but could leverage
cargo-release in the future

.github/pull_request_template.md

@@ -7,6 +7,19 @@ Thanks for sending a pull request!  Here are some tips for you:
   5. Be sure to keep the PR description updated to reflect all changes.
 -->
 
+<!--
+PR title formatting:
+This project uses conventional commits: https://www.conventionalcommits.org/
+
+Each PR corresponds to a commit on the `main` branch, with the title of the PR (typically) being
+used for the commit message on main. In order to ensure proper formatting in the CHANGELOG please
+ensure your PR title adheres to the conventional commit specification.
+
+Examples:
+- new feature PR: "feat: new API for snapshot.update()"
+- bugfix PR: "fix: correctly apply DV in read-table example"
+-->
+
 ## What changes are proposed in this pull request?
 <!--
 Please clarify what changes you are proposing and why the changes are needed.

acceptance/Cargo.toml

@@ -10,6 +10,9 @@ readme.workspace = true
 version.workspace = true
 rust-version.workspace = true
 
+[package.metadata.release]
+release = false
+
 [dependencies]
 arrow-array = { workspace = true }
 arrow-cast = { workspace = true }

cliff.toml

@@ -0,0 +1,70 @@
+# git-cliff configuration file. see https://git-cliff.org/docs/configuration
+
+[changelog]
+header = """
+# Changelog\n
+"""
+# Tera template
+body = """
+## [{{ version }}](https://github.com/delta-io/delta-kernel-rs/tree/{{ version }}/) ({{ timestamp | date(format="%Y-%m-%d") }})
+
+[Full Changelog](https://github.com/delta-io/delta-kernel-rs/compare/{{ previous.version }}...{{ version }})
+
+{% for group, commits in commits | group_by(attribute="group") %}
+    ### {{ group | striptags | trim | upper_first }}
+    {% for commit in commits %}
+        {{ loop.index }}. {% if commit.scope %}*({{ commit.scope }})* {% endif %}\
+            {{ commit.message | split(pat="\n") | first | upper_first | replace(from="(#", to="([#")\
+            | replace(from="0)", to="0])")\
+            | replace(from="1)", to="1])")\
+            | replace(from="2)", to="2])")\
+            | replace(from="3)", to="3])")\
+            | replace(from="4)", to="4])")\
+            | replace(from="5)", to="5])")\
+            | replace(from="6)", to="6])")\
+            | replace(from="7)", to="7])")\
+            | replace(from="8)", to="8])")\
+            | replace(from="9)", to="9])") }}\
+    {% endfor %}
+{% endfor %}
+{% for commit in commits %}
+{% set message = commit.message | split(pat="\n") | first %}\
+{% set pr = message | split(pat="(#") | last | split(pat=")") | first %}\
+[#{{ pr }}]: https://github.com/delta-io/delta-kernel-rs/pull/{{ pr }}\
+{% endfor %}\n\n\n
+"""
+footer = """
+"""
+# remove the leading and trailing s
+trim = true
+postprocessors = []
+
+[git]
+# parse the commits based on https://www.conventionalcommits.org
+conventional_commits = true
+# filter out the commits that are not conventional
+filter_unconventional = false
+# process each line of a commit as an individual commit
+split_commits = false
+# regex for preprocessing the commit messages
+commit_preprocessors = []
+# regex for parsing and grouping commits. note that e.g. both doc and docs are matched since we have
+# trim = true above.
+commit_parsers = [
+  { field = "github.pr_labels", pattern = "breaking-change", group = "<!-- 0 --> 🏗️ Breaking changes" },
+  { message = "^feat", group = "<!-- 1 -->🚀 Features / new APIs" },
+  { message = "^fix", group = "<!-- 2 -->🐛 Bug Fixes" },
+  { message = "^doc", group = "<!-- 3 -->📚 Documentation" },
+  { message = "^perf", group = "<!-- 4 -->⚡ Performance" },
+  { message = "^refactor", group = "<!-- 5 -->🚜 Refactor" },
+  { message = "^test", group = "<!-- 6 -->🧪 Testing" },
+  { message = "^chore|^ci", group = "<!-- 7 -->⚙️ Chores/CI" },
+  { message = "^revert", group = "<!-- 8 -->◀️ Revert" },
+  { message = ".*", group = "<!-- 9 -->Other" },
+]
+# filter out the commits that are not matched by commit parsers
+filter_commits = false
+# sort the tags topologically
+topo_order = false
+# sort the commits inside sections by oldest/newest order
+sort_commits = "oldest"

feature-tests/Cargo.toml

@@ -8,6 +8,9 @@ repository.workspace = true
 readme.workspace = true
 version.workspace = true
 
+[package.metadata.release]
+release = false
+
 [dependencies]
 delta_kernel = { path = "../kernel" }
 

ffi-proc-macros/Cargo.toml

@@ -10,6 +10,9 @@ readme.workspace = true
 rust-version.workspace = true
 version.workspace = true
 
+[package.metadata.release]
+release = false
+
 [lib]
 proc-macro = true
 

ffi/Cargo.toml

@@ -10,6 +10,9 @@ version.workspace = true
 rust-version.workspace = true
 build = "build.rs"
 
+[package.metadata.release]
+release = false
+
 [lib]
 crate-type = ["lib", "cdylib", "staticlib"]
 

kernel/Cargo.toml

@@ -15,6 +15,13 @@ rust-version.workspace = true
 [package.metadata.docs.rs]
 all-features = true
 
+[package.metadata.release]
+pre-release-replacements = [
+  {file="../README.md", search="delta_kernel = \"[a-z0-9\\.-]+\"", replace="delta_kernel = \"{{version}}\""},
+  {file="../README.md", search="version = \"[a-z0-9\\.-]+\"", replace="version = \"{{version}}\""},
+]
+pre-release-hook = ["git", "cliff", "--repository", "../", "--config", "../cliff.toml", "--unreleased", "--prepend", "../CHANGELOG.md", "--tag", "{{version}}" ]
+
 [dependencies]
 bytes = "1.7"
 chrono = { version = "0.4" }

kernel/examples/inspect-table/Cargo.toml

@@ -15,3 +15,6 @@ delta_kernel = { path = "../../../kernel", features = [
 ] }
 env_logger = "0.11.3"
 url = "2"
+
+[package.metadata.release]
+release = false

kernel/examples/read-table-changes/Cargo.toml

@@ -4,6 +4,9 @@ version = "0.1.0"
 edition = "2021"
 publish = false
 
+[package.metadata.release]
+release = false
+
 [dependencies]
 arrow-array = { workspace = true }
 arrow-schema = { workspace = true }

kernel/examples/read-table-multi-threaded/Cargo.toml

@@ -17,3 +17,6 @@ env_logger = "0.11.5"
 itertools = "0.13"
 spmc = "0.3.0"
 url = "2"
+
+[package.metadata.release]
+release = false

kernel/examples/read-table-single-threaded/Cargo.toml

@@ -16,3 +16,6 @@ delta_kernel = { path = "../../../kernel", features = [
 env_logger = "0.11.5"
 itertools = "0.13"
 url = "2"
+
+[package.metadata.release]
+release = false

release.sh

@@ -0,0 +1,170 @@
+#!/usr/bin/env bash
+
+###################################################################################################
+# USAGE:
+# 1. on a release branch: ./release.sh <version>
+# 2. on main branch (after merging release branch): ./release.sh
+###################################################################################################
+
+# This is a script to automate a large portion of the release process for the crates we publish to
+# crates.io. Currently only `delta_kernel` (in the kernel/ dir) and `delta_kernel_derive` (in the
+# derive-macros/ dir) are released.
+
+# Exit on error, undefined variables, and pipe failures
+set -euo pipefail
+
+# print commands before executing them for debugging
+# set -x
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+BLUE='\033[0;34m'
+NC='\033[0m' # no color
+
+log_info() { echo -e "${BLUE}[INFO]${NC} $1"; }
+log_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; }
+log_warning() { echo -e "${YELLOW}[WARNING]${NC} $1"; }
+log_error() { echo -e "${RED}[ERROR]${NC} $1"; exit 1; }
+
+check_requirements() {
+    log_info "Checking required tools..."
+
+    command -v cargo >/dev/null 2>&1 || log_error "cargo is required but not installed"
+    command -v git >/dev/null 2>&1 || log_error "git is required but not installed"
+    command -v cargo-release >/dev/null 2>&1 || log_error "cargo-release is required but not installed. Install with: cargo install cargo-release"
+    command -v git-cliff >/dev/null 2>&1 || log_error "git-cliff is required but not installed. Install with: cargo install git-cliff"
+    command -v jq >/dev/null 2>&1 || log_error "jq is required but not installed."
+
+    log_success "All required tools are available"
+}
+
+is_main_branch() {
+    local current_branch
+    current_branch=$(git rev-parse --abbrev-ref HEAD)
+    [[ "$current_branch" == "main" ]]
+}
+
+is_working_tree_clean() {
+    git diff --quiet && git diff --cached --quiet
+}
+
+# check if the version is already published on crates.io
+is_version_published() {
+    local crate_name="$1"
+    local version
+    version=get_current_version "$crate_name"
+
+    if [[ -z "$version" ]]; then
+        log_error "Could not find crate '$crate_name' in workspace"
+    fi
+
+    if cargo search "$crate_name" | grep -q "^$crate_name = \"$version\""; then
+        return 0
+    else
+        return 1
+    fi
+}
+
+# get current version from Cargo.toml
+get_current_version() {
+    local crate_name="$1"
+    cargo metadata --no-deps --format-version 1 | \
+        jq -r --arg name "$crate_name" '.packages[] | select(.name == $name) | .version'
+}
+
+# Prompt user for confirmation
+confirm() {
+    local prompt="$1"
+    local response
+
+    echo -e -n "${YELLOW}${prompt} [y/N]${NC} "
+    read -r response
+
+    [[ "$response" =~ ^[Yy] ]]
+}
+
+# handle release branch workflow (CHANGELOG updates, README updates, PR to main)
+handle_release_branch() {
+    local version="$1"
+
+    log_info "Starting release preparation for version $version..."
+
+    # Update CHANGELOG and README
+    log_info "Updating CHANGELOG.md and README.md..."
+    if ! cargo release --workspace "$version" --no-publish --no-push --no-tag --execute; then
+        log_error "Failed to update CHANGELOG and README"
+    fi
+
+    if confirm "Print diff of CHANGELOG/README changes?"; then
+        git diff --stat HEAD^
+        git diff HEAD^
+    fi
+
+    if confirm "Would you like to push these changes to 'origin' remote?"; then
+        local current_branch
+        current_branch=$(git rev-parse --abbrev-ref HEAD)
+
+        log_info "Pushing changes to remote..."
+        git push origin "$current_branch"
+
+        if confirm "Would you like to create a PR to merge this release into 'main'?"; then
+            if command -v gh >/dev/null 2>&1; then
+                gh pr create --title "release $version" --body "release $version"
+                log_success "PR created successfully"
+            else
+                log_warning "GitHub CLI not found. Please create a PR manually."
+            fi
+        fi
+    fi
+}
+
+# Handle main branch workflow (publish and tag)
+handle_main_branch() {
+    # could potentially just use full 'cargo release' command here
+    publish "delta_kernel_derive" "$current_version"
+    publish "delta_kernel" "$current_version"
+
+    if confirm "Would you like to tag this release?"; then
+        log_info "Tagging release $current_version..."
+        git tag -a "v$current_version" -m "Release $current_version"
+        git push upstream "v$current_version"
+        log_success "Tagged release $current_version"
+    fi
+}
+
+publish() {
+    local crate_name="$1"
+    local current_version
+    current_version=$(get_current_version "$crate_name")
+
+    if is_version_published "delta_kernel_derive"; then
+        log_error "delta_kernel_derive version $current_version is already published to crates.io"
+    fi
+    log_info "[DRY RUN] Publishing $crate_name version $version to crates.io..."
+    if ! cargo publish --dry-run -p "$crate_name"; then
+        log_error "Failed to publish $crate_name to crates.io"
+    fi
+
+    if confirm "Dry run complete. Continue with publishing?"; then
+        log_info "Publishing $crate_name version $version to crates.io..."
+        if ! cargo publish -p "$crate_name"; then
+            log_error "Failed to publish $crate_name to crates.io"
+        fi
+        log_success "Successfully published $crate_name version $version to crates.io"
+    fi
+}
+
+check_requirements
+
+if is_main_branch; then
+    if [[ $# -ne 0 ]]; then
+        log_error "Version argument not expected on main branch\nUsage: $0"
+    fi
+    handle_main_branch
+else
+    if [[ $# -ne 1 ]]; then
+        log_error "Version argument required when on release branch\nUsage: $0 <version>"
+    fi
+    handle_release_branch "$1"
+fi

release.toml

@@ -0,0 +1,3 @@
+tag = false
+publish = false
+pre-release-commit-message = "release {{version}}"

test-utils/Cargo.toml

@@ -8,6 +8,9 @@ repository.workspace = true
 readme.workspace = true
 version.workspace = true
 
+[package.metadata.release]
+release = false
+
 [dependencies]
 arrow-array = { workspace = true, features = ["chrono-tz"] }
 arrow-schema = { workspace = true }