diff --git a/.devcontainer/.dockerignore b/.devcontainer/.dockerignore new file mode 100644 index 000000000000..b43bf86b50fd --- /dev/null +++ b/.devcontainer/.dockerignore @@ -0,0 +1 @@ +README.md diff --git a/.devcontainer/.ssh.sh b/.devcontainer/.ssh.sh new file mode 100644 index 000000000000..513301df869b --- /dev/null +++ b/.devcontainer/.ssh.sh @@ -0,0 +1,73 @@ +#!/bin/sh +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +if [ -d ~/.ssh ]; then + if echo "$(mountpoint ~/.ssh)" | grep -q "is a mountpoint"; then + # ~/.ssh is a bind mount from the host + return 0; + fi + echo "$(/bin/ls -a /mnt/ssh 2>/dev/null)" > /tmp/ls_mnt_ssh + echo "$(/bin/ls -a ~/.ssh 2>/dev/null)" > /tmp/ls_ssh + echo "$(/bin/ls -a /tmp/.ssh 2>/dev/null)" > /tmp/ls_tmp_ssh + if [ -d /mnt/ssh ] && [ -z "$(comm -3 /tmp/ls_mnt_ssh /tmp/ls_ssh)" ]; then + # /mnt/ssh and ~/.ssh are the same in terms of file names. + rm /tmp/ls_mnt_ssh + rm /tmp/ls_ssh + rm /tmp/ls_tmp_ssh + return 0; + fi + if [ -d /tmp/.ssh ] && [ -z "$(comm -3 /tmp/ls_tmp_ssh /tmp/ls_ssh)" ]; then + # Retro-compatibility: /tmp/.ssh and ~/.ssh are the same in terms of file names. + rm /tmp/ls_mnt_ssh + rm /tmp/ls_ssh + rm /tmp/ls_tmp_ssh + return 0; + fi + rm /tmp/ls_mnt_ssh + rm /tmp/ls_ssh + rm /tmp/ls_tmp_ssh +fi + +if [ -d /tmp/.ssh ]; then + # Retro-compatibility + echo "Copying content of /tmp/.ssh to ~/.ssh" + mkdir -p ~/.ssh + cp -r /tmp/.ssh/* ~/.ssh/ + chmod 600 ~/.ssh/* + chmod 644 ~/.ssh/*.pub &> /dev/null + return 0 +fi +if [ ! -d /mnt/ssh ]; then + echo "No bind mounted ssh directory found (~/.ssh, /tmp/.ssh, /mnt/ssh), exiting" + return 0 +fi + +if [ "$(stat -c '%U' /mnt/ssh)" != "UNKNOWN" ]; then + echo "Unix host detected, symlinking /mnt/ssh to ~/.ssh" + rm -rf ~/.ssh + ln -s /mnt/ssh ~/.ssh + chmod 600 ~/.ssh/* + chmod 644 ~/.ssh/*.pub &> /dev/null + return 0 +fi + +echo "Windows host detected, copying content of /mnt/ssh to ~/.ssh" +mkdir -p ~/.ssh +cp -rf /mnt/ssh/* ~/.ssh/ +chmod 600 ~/.ssh/* diff --git a/.devcontainer/.welcome.sh b/.devcontainer/.welcome.sh new file mode 100644 index 000000000000..3de468b24a9b --- /dev/null +++ b/.devcontainer/.welcome.sh @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +rustup show \ No newline at end of file diff --git a/.devcontainer/.zshrc-specific b/.devcontainer/.zshrc-specific new file mode 100644 index 000000000000..08c3b4ba2cc6 --- /dev/null +++ b/.devcontainer/.zshrc-specific @@ -0,0 +1,5 @@ +# This file should NOT interact with stdout nor stdin! + +plugins=(git colorize rust rustup docker docker-compose kubectl) + +export RUSTC_WRAPPER=sccache \ No newline at end of file diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 9dd627b01abe..912124838086 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,13 +1,85 @@ -FROM rust:bookworm +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. -RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - # Remove imagemagick due to https://security-tracker.debian.org/tracker/CVE-2019-10131 - && apt-get purge -y imagemagick imagemagick-6-common +# Dockerfile based on https://github.com/qdm12/rustdevcontainer/tree/main/.devcontainer +ARG BASEDEV_VERSION=v0.25.0 + +FROM qmcgaw/basedevcontainer:${BASEDEV_VERSION}-debian +ARG CREATED +ARG COMMIT +ARG VERSION=local +LABEL \ + org.opencontainers.image.authors="quentin.mcgaw@gmail.com" \ + org.opencontainers.image.created=$CREATED \ + org.opencontainers.image.version=$VERSION \ + org.opencontainers.image.revision=$COMMIT \ + org.opencontainers.image.url="https://github.com/qdm12/rustdevcontainer" \ + org.opencontainers.image.documentation="https://github.com/qdm12/rustdevcontainer" \ + org.opencontainers.image.source="https://github.com/qdm12/rustdevcontainer" \ + org.opencontainers.image.title="Rust Debian Dev container" \ + org.opencontainers.image.description="Rust development container" +WORKDIR /apache_datafusion + +# Install Rust for the correct CPU architecture +ARG RUST_VERSION=1.85.0 +ARG RUSTUP_INIT_VERSION=1.27.1 +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH +RUN arch="$(uname -m)" && \ + case "$arch" in \ + x86_64) rustArch='x86_64-unknown-linux-gnu'; rustupSha256='6aeece6993e902708983b209d04c0d1dbb14ebb405ddb87def578d41f920f56d' ;; \ + aarch64) rustArch='aarch64-unknown-linux-gnu'; rustupSha256='1cffbf51e63e634c746f741de50649bbbcbd9dbe1de363c9ecef64e278dba2b2' ;; \ + *) echo >&2 "unsupported architecture: ${dpkgArch}"; exit 1 ;; \ + esac && \ + wget -qO /tmp/rustup-init "https://static.rust-lang.org/rustup/archive/${RUSTUP_INIT_VERSION}/${rustArch}/rustup-init" && \ + echo "${rustupSha256} /tmp/rustup-init" | sha256sum -c - && \ + chmod +x /tmp/rustup-init && \ + /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain ${RUST_VERSION} --default-host ${rustArch} && \ + rm /tmp/rustup-init && \ + chmod -R a+w ${RUSTUP_HOME} ${CARGO_HOME} + +# Install: +# - unzip libxext6 libxext-dev libxtst-dev libxi-dev freetype* procps gdb - required by intellij (https://www.jetbrains.com/help/idea/prerequisites-for-dev-containers.html) +# - build-essential, libc6-dev, clang required by Rust +# - musl-tools required for static binaries +# - sccache used to speed up cargo builds +# - mold used to speed up linking +# - vim because, well, vi +# - protobuf-compiler for proto file compilation +# - pkg-config, libssl-dev required for build +# - npm - required to run prettier +# - cmake - to build a dep +RUN export DEBIAN_FRONTEND=noninteractive && \ + apt-get update && \ + apt-get -y upgrade && \ + apt-get -y install unzip libxext6 libxext-dev libxtst-dev libxi-dev freetype* procps gdb build-essential clang libc6-dev musl-tools sccache mold vim protobuf-compiler pkg-config libssl-dev npm cmake && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +# Install Rust tooling +ARG RUST_ANALYZER_VERSION=2024-07-01 +RUN wget -qO- "https://github.com/rust-analyzer/rust-analyzer/releases/download/${RUST_ANALYZER_VERSION}/rust-analyzer-$(uname -m)-unknown-linux-gnu.gz" | \ + gunzip > /usr/local/bin/rust-analyzer && \ + chmod 500 /usr/local/bin/rust-analyzer +RUN rustup component add clippy rustfmt llvm-tools-preview && cargo install cargo-nextest + +# Shell setup +COPY .zshrc-specific .welcome.sh .ssh.sh /root/ +RUN chmod 750 ~/.ssh.sh && mkdir ~/.zfunc && rustup completions zsh > ~/.zfunc/_rustup && mkdir -p ~/.ssh && chmod 600 ~/.ssh -# Add protoc -# https://datafusion.apache.org/contributor-guide/getting_started.html#protoc-installation -RUN curl -LO https://github.com/protocolbuffers/protobuf/releases/download/v25.1/protoc-25.1-linux-x86_64.zip \ - && unzip protoc-25.1-linux-x86_64.zip -d $HOME/.local \ - && rm protoc-25.1-linux-x86_64.zip -ENV PATH="$PATH:$HOME/.local/bin" \ No newline at end of file diff --git a/.devcontainer/README.md b/.devcontainer/README.md new file mode 100644 index 000000000000..5586b008fbba --- /dev/null +++ b/.devcontainer/README.md @@ -0,0 +1,81 @@ + + +# Apache Datafusion development container + +This development container can be used with RustRover or IntelliJ to create and run a docker image for development +purposes. It is based on this [Rust dev container](https://github.com/qdm12/rustdevcontainer/tree/main/.devcontainer) +and it should work on Linux, Windows and OSX though currently on OSX IntelliJ required Docker and it will not work with +podman. + +## Requirements + +- [Docker](https://www.docker.com/products/docker-desktop) installed and running +- [Docker Compose](https://docs.docker.com/compose/install/) installed + +## Setup prior to running the dev container: + +1. Create the following files on your host if you don't have them: + + ```shell + touch ~/.gitconfig ~/.zsh_history + ``` +1. Configure git ssh url rewrite in ~/.gitconfig on your host by adding the following: + ```shell + [url "git@github.com:"] + insteadOf = https://github.com/ + ``` + Note that the development container will create the empty directories `~/.docker`, `~/.ssh` if you don't have them. + +1. **For Docker on OSX or Windows without WSL**: ensure your home directory `~` is accessible by Docker. + +1. In RustRover/IntelliJ -> File -> Remote Development... -> Dev Containers -> New Dev Container -> Select From VCS + Project and enter the git url for this repository in the `git repository:` field. VSCode should also work but is + untested. +1. RustRover/IntelliJ will proceed to build the container and run it. Note that the docker-compose.yml mounts the ~/.ssh +into the container so that git commands will function correctly (assuming they work on your local machine). + +## Customization + +### Customize the image + +You can make changes to the [Dockerfile](Dockerfile) and then rebuild the image. + +### Publish a port + +To access a port from your host to your development container, publish a port +in [docker-compose.yml](docker-compose.yml). If you use vscode you can also now do it directly with VSCode without +restarting the container. + +### Run other services + +1. Modify [docker-compose.yml](docker-compose.yml) to launch other services at the same time as this development + container, such as a test database: + + ```yml + database: + image: postgres + restart: always + environment: + POSTGRES_PASSWORD: password + ``` + +1. In [devcontainer.json](devcontainer.json), change the line `"runServices": ["datafusion"],` + to `"runServices": ["datafusion", "database"],`. +1. Rebuild the container. diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 1af22306ed8c..c18c5dcd0c9e 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,16 +1,39 @@ { - "build": { - "dockerfile": "./Dockerfile", - "context": "." - }, - "customizations": { - "vscode": { - "extensions": [ - "rust-lang.rust-analyzer" - ] - } - }, - "features": { - "ghcr.io/devcontainers/features/rust:1": "latest" +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + "name": "datafusion", + "dockerComposeFile": [ + "docker-compose.yml" + ], + "service": "datafusion", + "runServices": [ + "datafusion" + ], + "shutdownAction": "stopCompose", + "postCreateCommand": "~/.ssh.sh", + "workspaceFolder": "/apache_datafusion", + // "overrideCommand": "", + "customizations": { + "jetbrains": { + "backend": "RustRover", + "settings": { + // rustrover doesn't seem to have any settings available that I could find right now + // ideally, we would change the linter to clippy here among other things + }, } -} + } +} \ No newline at end of file diff --git a/.devcontainer/docker-compose.yml b/.devcontainer/docker-compose.yml new file mode 100644 index 000000000000..ed4f518c13a1 --- /dev/null +++ b/.devcontainer/docker-compose.yml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +services: + datafusion: + build: . + volumes: + - ../:/apache_datafusion + # Docker socket to access Docker server + - /var/run/docker.sock:/var/run/docker.sock + # SSH directory for Linux, OSX and WSL + # On Linux and OSX, a symlink /mnt/ssh <-> ~/.ssh is + # created in the container. On Windows, files are copied + # from /mnt/ssh to ~/.ssh to fix permissions. + - ~/.ssh:/mnt/ssh + # Shell history persistence + #- ~/.zsh_history:/root/.zsh_history + # Git config + - ~/.gitconfig:/root/.gitconfig + environment: + - TZ= + # Needed for debugging + # cap_add: + # - SYS_PTRACE + # security_opt: + # - seccomp:unconfined + entrypoint: [ "zsh", "-c", "while sleep 1000; do :; done" ]