devcontainer.

.devcontainer/.dockerignore

@@ -0,0 +1 @@
+README.md

.devcontainer/.ssh.sh

@@ -0,0 +1,73 @@
+#!/bin/sh
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+if [ -d ~/.ssh ]; then
+  if echo "$(mountpoint ~/.ssh)" | grep -q "is a mountpoint"; then
+    # ~/.ssh is a bind mount from the host
+    return 0;
+  fi
+  echo "$(/bin/ls -a /mnt/ssh 2>/dev/null)" > /tmp/ls_mnt_ssh
+  echo "$(/bin/ls -a ~/.ssh 2>/dev/null)" > /tmp/ls_ssh
+  echo "$(/bin/ls -a /tmp/.ssh 2>/dev/null)" > /tmp/ls_tmp_ssh
+  if [ -d /mnt/ssh ] && [ -z "$(comm -3 /tmp/ls_mnt_ssh /tmp/ls_ssh)" ]; then
+    # /mnt/ssh and ~/.ssh are the same in terms of file names.
+    rm /tmp/ls_mnt_ssh
+    rm /tmp/ls_ssh
+    rm /tmp/ls_tmp_ssh
+    return 0;
+  fi
+  if [ -d /tmp/.ssh ] && [ -z "$(comm -3 /tmp/ls_tmp_ssh /tmp/ls_ssh)" ]; then
+    # Retro-compatibility: /tmp/.ssh and ~/.ssh are the same in terms of file names.
+    rm /tmp/ls_mnt_ssh
+    rm /tmp/ls_ssh
+    rm /tmp/ls_tmp_ssh
+    return 0;
+  fi
+  rm /tmp/ls_mnt_ssh
+  rm /tmp/ls_ssh
+  rm /tmp/ls_tmp_ssh
+fi
+
+if [ -d /tmp/.ssh ]; then
+  # Retro-compatibility
+  echo "Copying content of /tmp/.ssh to ~/.ssh"
+  mkdir -p ~/.ssh
+  cp -r /tmp/.ssh/* ~/.ssh/
+  chmod 600 ~/.ssh/*
+  chmod 644 ~/.ssh/*.pub &> /dev/null
+  return 0
+fi
+if [ ! -d /mnt/ssh ]; then 
+  echo "No bind mounted ssh directory found (~/.ssh, /tmp/.ssh, /mnt/ssh), exiting" 
+  return 0 
+fi 
+
+if [ "$(stat -c '%U' /mnt/ssh)" != "UNKNOWN" ]; then 
+  echo "Unix host detected, symlinking /mnt/ssh to ~/.ssh" 
+  rm -rf ~/.ssh
+  ln -s /mnt/ssh ~/.ssh
+  chmod 600 ~/.ssh/*
+  chmod 644 ~/.ssh/*.pub &> /dev/null
+  return 0
+fi 
+
+echo "Windows host detected, copying content of /mnt/ssh to ~/.ssh" 
+mkdir -p ~/.ssh 
+cp -rf /mnt/ssh/* ~/.ssh/ 
+chmod 600 ~/.ssh/*

.devcontainer/.welcome.sh

@@ -0,0 +1,18 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+rustup show

.devcontainer/.zshrc-specific

@@ -0,0 +1,5 @@
+# This file should NOT interact with stdout nor stdin!
+
+plugins=(git colorize rust rustup docker docker-compose kubectl)
+
+export RUSTC_WRAPPER=sccache

.devcontainer/Dockerfile

@@ -1,13 +1,85 @@
-FROM rust:bookworm
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
-RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
-    # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131
-    && apt-get purge -y imagemagick imagemagick-6-common
+# Dockerfile based on https://github.com/qdm12/rustdevcontainer/tree/main/.devcontainer
+ARG BASEDEV_VERSION=v0.25.0
+
+FROM qmcgaw/basedevcontainer:${BASEDEV_VERSION}-debian
+ARG CREATED
+ARG COMMIT
+ARG VERSION=local
+LABEL \
+    org.opencontainers.image.authors="quentin.mcgaw@gmail.com" \
+    org.opencontainers.image.created=$CREATED \
+    org.opencontainers.image.version=$VERSION \
+    org.opencontainers.image.revision=$COMMIT \
+    org.opencontainers.image.url="https://github.com/qdm12/rustdevcontainer" \
+    org.opencontainers.image.documentation="https://github.com/qdm12/rustdevcontainer" \
+    org.opencontainers.image.source="https://github.com/qdm12/rustdevcontainer" \
+    org.opencontainers.image.title="Rust Debian Dev container" \
+    org.opencontainers.image.description="Rust development container"
+WORKDIR /apache_datafusion
+
+# Install Rust for the correct CPU architecture
+ARG RUST_VERSION=1.85.0
+ARG RUSTUP_INIT_VERSION=1.27.1
+ENV RUSTUP_HOME=/usr/local/rustup \
+    CARGO_HOME=/usr/local/cargo \
+    PATH=/usr/local/cargo/bin:$PATH
+RUN arch="$(uname -m)" && \
+    case "$arch" in \
+    x86_64) rustArch='x86_64-unknown-linux-gnu'; rustupSha256='6aeece6993e902708983b209d04c0d1dbb14ebb405ddb87def578d41f920f56d' ;; \
+    aarch64) rustArch='aarch64-unknown-linux-gnu'; rustupSha256='1cffbf51e63e634c746f741de50649bbbcbd9dbe1de363c9ecef64e278dba2b2' ;; \
+    *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \
+    esac && \
+    wget -qO /tmp/rustup-init "https://static.rust-lang.org/rustup/archive/${RUSTUP_INIT_VERSION}/${rustArch}/rustup-init" && \
+    echo "${rustupSha256}  /tmp/rustup-init" | sha256sum -c - && \
+    chmod +x /tmp/rustup-init && \
+    /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain ${RUST_VERSION} --default-host ${rustArch} && \
+    rm /tmp/rustup-init && \
+    chmod -R a+w ${RUSTUP_HOME} ${CARGO_HOME}
+
+# Install:
+# - unzip libxext6 libxext-dev libxtst-dev libxi-dev freetype* procps gdb - required by intellij (https://www.jetbrains.com/help/idea/prerequisites-for-dev-containers.html)
+# - build-essential, libc6-dev, clang required by Rust
+# - musl-tools required for static binaries
+# - sccache used to speed up cargo builds
+# - mold used to speed up linking
+# - vim because, well, vi
+# - protobuf-compiler for proto file compilation
+# - pkg-config, libssl-dev required for build
+# - npm - required to run prettier
+# - cmake - to build a dep
+RUN export DEBIAN_FRONTEND=noninteractive && \
+    apt-get update && \
+    apt-get -y upgrade && \
+    apt-get -y install unzip libxext6 libxext-dev libxtst-dev libxi-dev freetype* procps gdb build-essential clang libc6-dev musl-tools sccache mold vim protobuf-compiler pkg-config libssl-dev npm cmake && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Install Rust tooling
+ARG RUST_ANALYZER_VERSION=2024-07-01
+RUN wget -qO- "https://github.com/rust-analyzer/rust-analyzer/releases/download/${RUST_ANALYZER_VERSION}/rust-analyzer-$(uname -m)-unknown-linux-gnu.gz" | \
+    gunzip > /usr/local/bin/rust-analyzer && \
+    chmod 500 /usr/local/bin/rust-analyzer
+RUN rustup component add clippy rustfmt llvm-tools-preview && cargo install cargo-nextest
+
+# Shell setup
+COPY .zshrc-specific .welcome.sh .ssh.sh /root/
+RUN chmod 750 ~/.ssh.sh && mkdir ~/.zfunc && rustup completions zsh > ~/.zfunc/_rustup && mkdir -p ~/.ssh && chmod 600 ~/.ssh
 
-# Add protoc
-# https://datafusion.apache.org/contributor-guide/getting_started.html#protoc-installation
-RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v25.1/protoc-25.1-linux-x86_64.zip \
-    && unzip protoc-25.1-linux-x86_64.zip -d $HOME/.local \
-    && rm protoc-25.1-linux-x86_64.zip
 
-ENV PATH="$PATH:$HOME/.local/bin"

.devcontainer/README.md

@@ -0,0 +1,81 @@
+<!---
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing,
+  software distributed under the License is distributed on an
+  "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+  KIND, either express or implied.  See the License for the
+  specific language governing permissions and limitations
+  under the License.
+-->
+
+# Apache Datafusion development container
+
+This development container can be used with RustRover or IntelliJ to create and run a docker image for development
+purposes. It is based on this [Rust dev container](https://github.com/qdm12/rustdevcontainer/tree/main/.devcontainer)
+and it should work on Linux, Windows and OSX though currently on OSX IntelliJ required Docker and it will not work with
+podman.
+
+## Requirements
+
+- [Docker](https://www.docker.com/products/docker-desktop) installed and running
+- [Docker Compose](https://docs.docker.com/compose/install/) installed
+
+## Setup prior to running the dev container:
+
+1. Create the following files on your host if you don't have them:
+
+    ```shell
+    touch ~/.gitconfig ~/.zsh_history
+    ```
+1. Configure git ssh url rewrite in ~/.gitconfig on your host by adding the following:
+   ```shell
+   [url "git@github.com:"]
+     insteadOf = https://github.com/
+   ```
+   Note that the development container will create the empty directories `~/.docker`, `~/.ssh` if you don't have them.
+
+1. **For Docker on OSX or Windows without WSL**: ensure your home directory `~` is accessible by Docker.
+
+1. In RustRover/IntelliJ -> File -> Remote Development... -> Dev Containers -> New Dev Container -> Select From VCS
+   Project and enter the git url for this repository in the `git repository:` field. VSCode should also work but is
+   untested.
+1. RustRover/IntelliJ will proceed to build the container and run it. Note that the docker-compose.yml mounts the ~/.ssh
+into the container so that git commands will function correctly (assuming they work on your local machine).
+
+## Customization
+
+### Customize the image
+
+You can make changes to the [Dockerfile](Dockerfile) and then rebuild the image.
+
+### Publish a port
+
+To access a port from your host to your development container, publish a port
+in [docker-compose.yml](docker-compose.yml). If you use vscode you can also now do it directly with VSCode without
+restarting the container.
+
+### Run other services
+
+1. Modify [docker-compose.yml](docker-compose.yml) to launch other services at the same time as this development
+   container, such as a test database:
+
+    ```yml
+      database:
+        image: postgres
+        restart: always
+        environment:
+          POSTGRES_PASSWORD: password
+    ```
+
+1. In [devcontainer.json](devcontainer.json), change the line `"runServices": ["datafusion"],`
+   to `"runServices": ["datafusion", "database"],`.
+1. Rebuild the container.

.devcontainer/devcontainer.json

@@ -1,16 +1,39 @@
 {
-    "build": {
-        "dockerfile": "./Dockerfile",
-        "context": "."
-    },
-    "customizations": {
-        "vscode": {
-            "extensions": [
-                "rust-lang.rust-analyzer"
-            ]
-        }
-    },
-    "features": {
-        "ghcr.io/devcontainers/features/rust:1": "latest"
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+  "name": "datafusion",
+  "dockerComposeFile": [
+    "docker-compose.yml"
+  ],
+  "service": "datafusion",
+  "runServices": [
+    "datafusion"
+  ],
+  "shutdownAction": "stopCompose",
+  "postCreateCommand": "~/.ssh.sh",
+  "workspaceFolder": "/apache_datafusion",
+  // "overrideCommand": "",
+  "customizations": {
+    "jetbrains": {
+      "backend": "RustRover",
+      "settings": {
+        // rustrover doesn't seem to have any settings available that I could find right now
+        // ideally, we would change the linter to clippy here among other things
+      },
     }
-}
+  }
+}

.devcontainer/docker-compose.yml

@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+services:
+  datafusion:
+    build: .
+    volumes:
+      - ../:/apache_datafusion
+      # Docker socket to access Docker server
+      - /var/run/docker.sock:/var/run/docker.sock
+      # SSH directory for Linux, OSX and WSL
+      # On Linux and OSX, a symlink /mnt/ssh <-> ~/.ssh is
+      # created in the container. On Windows, files are copied
+      # from /mnt/ssh to ~/.ssh to fix permissions.
+      - ~/.ssh:/mnt/ssh
+      # Shell history persistence
+      #- ~/.zsh_history:/root/.zsh_history
+      # Git config
+      - ~/.gitconfig:/root/.gitconfig
+    environment:
+      - TZ=
+    # Needed for debugging
+    # cap_add:
+    #   - SYS_PTRACE
+    # security_opt:
+    #   - seccomp:unconfined
+    entrypoint: [ "zsh", "-c", "while sleep 1000; do :; done" ]