diff --git a/.gitignore b/.gitignore index 3bdff6e..bc36404 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,7 @@ erl_crash.dump *.ez # Ignore package tarball (built via "mix hex.build"). -eips-*.tar +vix-*.tar /.ccls-cache @@ -52,4 +52,10 @@ checksum.exs # mise mise.toml -/cache \ No newline at end of file +# Local development artifacts +/.libvips/ +/.tmux/ +/toolchains/ +/.dexter.* + +/cache diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 2f2798d..32a53d7 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -5,6 +5,7 @@ This document provides comprehensive guidance for Vix development, testing, and ## Table of Contents - [Development Environment Setup](#development-environment-setup) +- [Building libvips from Source](#building-libvips-from-source) - [Build System](#build-system) - [Toolchain Management](#toolchain-management) - [Testing and Quality Assurance](#testing-and-quality-assurance) @@ -26,16 +27,48 @@ Vix supports three compilation modes controlled by the `VIX_COMPILATION_MODE` en 1. **`PRECOMPILED_NIF_AND_LIBVIPS`** (default) - Use precompiled NIFs and libvips 2. **`PRECOMPILED_LIBVIPS`** - Compile NIFs locally, use precompiled libvips -3. **`PLATFORM_PROVIDED_LIBVIPS`** - Use system-provided libvips +3. **`PLATFORM_PROVIDED_LIBVIPS`** - Use a libvips installation resolved via `pkg-config` ```bash -# Use system libvips (requires libvips-dev package) +# Use the system libvips installation (requires libvips-dev package) export VIX_COMPILATION_MODE=PLATFORM_PROVIDED_LIBVIPS +# Use a specific libvips install prefix instead of the system default +export VIX_COMPILATION_MODE=PLATFORM_PROVIDED_LIBVIPS +export VIX_LIBVIPS_PREFIX=/path/to/libvips/prefix + # Force precompiled libvips build export VIX_COMPILATION_MODE=PRECOMPILED_LIBVIPS ``` +`VIX_LIBVIPS_PREFIX` is the libvips install prefix Vix should use, for example the path passed to Meson with `--prefix`. + +## Building libvips from Source + +For development against a newer libvips without packaging a custom binary, developers can use `scripts/build_libvips.sh` as a starting point to build libvips. + +It builds a `libvips` source archive into a private prefix using the libraries available on the local machine. + +The helper script expects `curl`, `tar`, `meson`, `ninja`, `pkg-config`, and a working C/C++ compiler. + +```bash +# Build a tagged release +./scripts/build_libvips.sh --ref vX.X.X +export VIX_LIBVIPS_PREFIX="$(pwd)/.libvips/vX.X.X" +export VIX_COMPILATION_MODE=PLATFORM_PROVIDED_LIBVIPS + +mix test + +# Build the current upstream development branch +./scripts/build_libvips.sh --ref master +export VIX_LIBVIPS_PREFIX="$(pwd)/.libvips/master" +export VIX_COMPILATION_MODE=PLATFORM_PROVIDED_LIBVIPS +``` + +If `--ref` is omitted, the script builds the latest upstream tagged release. + +`VIX_LIBVIPS_PREFIX` must point to the exact libvips install prefix you want Vix to use. + ### Initial Setup ```bash @@ -66,7 +99,7 @@ make deep_clean # Full clean including precompiled libvips make clean_precompiled_libvips # Remove only precompiled libvips # Debug and verbose builds -make debug # Show build configuration (from c_src/) +make debug # Show the native build configuration make V=1 all # Verbose compilation output ``` @@ -75,10 +108,12 @@ make V=1 all # Verbose compilation output Build behavior is controlled by several environment variables: - `VIX_COMPILATION_MODE` - Compilation strategy -- `LIBVIPS_VERSION` - Override default libvips version +- `VIX_LIBVIPS_PREFIX` - Custom libvips install prefix for `PLATFORM_PROVIDED_LIBVIPS` +- `LIBVIPS_VERSION` - Override default precompiled libvips version - `CC_PRECOMPILER_CURRENT_TARGET` - Override target platform - `ELIXIR_MAKE_CACHE_DIR` - Cache directory for precompiled binaries + ## Toolchain Management ### Musl Toolchain System @@ -279,8 +314,8 @@ export VIX_COMPILATION_MODE=PRECOMPILED_LIBVIPS make deep_clean make all -# Check build configuration -cd c_src && make debug +# Inspect the active native build configuration +make debug ``` **"Checksum verification failed"** @@ -306,6 +341,7 @@ tar -xzf x86_64-linux-musl-cross.tgz - Use `ELIXIR_MAKE_CACHE_DIR="$(pwd)/cache"` to cache precompiled binaries locally - Set `VIX_COMPILATION_MODE=PLATFORM_PROVIDED_LIBVIPS` for faster iteration during development -- Run `make debug` from `c_src/` directory to see detailed build configuration +- Use `VIX_LIBVIPS_PREFIX` when testing a specific local libvips build +- Run `make debug` to inspect the effective native build configuration - Use `mix test --trace` for verbose test output - Check GitHub Actions logs for CI build failures diff --git a/Makefile b/Makefile index d379152..c0cbd0b 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,11 @@ all: compile # Main compilation target compile: +ifdef ERTS_INCLUDE_DIR @$(MAKE) -C c_src all +else + @mix compile +endif # Mix compilation (called by elixir_make) calling_from_make: @@ -14,12 +18,22 @@ calling_from_make: # Clean targets clean: +ifdef ERTS_INCLUDE_DIR @$(MAKE) -C c_src clean +else + @mix clean +endif clean_precompiled_libvips: @$(MAKE) -C c_src clean_precompiled_libvips -deep_clean: clean_precompiled_libvips +deep_clean: +ifdef ERTS_INCLUDE_DIR + @$(MAKE) -C c_src clean_precompiled_libvips +else + @mix clean + @$(MAKE) -C c_src clean_precompiled_libvips +endif # Development targets test: @@ -31,9 +45,14 @@ format: lint: mix credo -dialyz: +dialyxir: mix dialyxir +dialyz: dialyxir + +debug: + @$(MAKE) -C c_src debug + # Help target help: @echo "Available targets:" @@ -44,7 +63,8 @@ help: @echo " test - Run tests" @echo " format - Format Elixir code" @echo " lint - Run Credo linter" - @echo " dialyz - Run Dialyzer type checking" + @echo " dialyxir/dialyz - Run Dialyzer type checking" + @echo " debug - Show native build configuration" @echo " help - Show this help" -.PHONY: all compile clean clean_precompiled_libvips deep_clean calling_from_make test format lint dialyz help +.PHONY: all compile clean clean_precompiled_libvips deep_clean calling_from_make test format lint dialyxir dialyz debug help diff --git a/README.md b/README.md index 2d96712..bad25c4 100644 --- a/README.md +++ b/README.md @@ -144,7 +144,18 @@ Want to use your system's libvips? Set before compilation: export VIX_COMPILATION_MODE=PLATFORM_PROVIDED_LIBVIPS ``` -See [libvips installation guide](https://www.libvips.org/install.html) for more details. +Want to use a custom libvips install prefix instead of the system default? Set: + +```bash +export VIX_COMPILATION_MODE=PLATFORM_PROVIDED_LIBVIPS +export VIX_LIBVIPS_PREFIX=/path/to/libvips/prefix +``` + +`VIX_LIBVIPS_PREFIX` is the libvips install prefix Vix should use, for example the path passed to Meson with `--prefix`. + +From a Git checkout, developers can use `scripts/build_libvips.sh` as a starting point to build libvips. + +See [libvips installation guide](https://www.libvips.org/install.html) and [DEVELOPMENT.md](DEVELOPMENT.md) for more details. ## Documentation & Resources diff --git a/c_src/Makefile b/c_src/Makefile index 503d785..677ad05 100644 --- a/c_src/Makefile +++ b/c_src/Makefile @@ -3,12 +3,17 @@ SHELL = /bin/sh # Build configuration MIX_APP_PATH ?= .. +PROJECT_ROOT := $(abspath $(CURDIR)/..) PREFIX = $(MIX_APP_PATH)/priv VIX = $(PREFIX)/vix.so # Compilation mode configuration VIX_COMPILATION_MODE ?= PRECOMPILED_NIF_AND_LIBVIPS VIX_COMPILATION_MODE := $(shell echo "$(VIX_COMPILATION_MODE)" | tr '[:lower:]' '[:upper:]') +VALID_COMPILATION_MODES := PRECOMPILED_NIF_AND_LIBVIPS PRECOMPILED_LIBVIPS PLATFORM_PROVIDED_LIBVIPS +ifeq ($(filter $(VIX_COMPILATION_MODE),$(VALID_COMPILATION_MODES)),) +$(error Invalid VIX_COMPILATION_MODE '$(VIX_COMPILATION_MODE)'. Valid modes: $(VALID_COMPILATION_MODES)) +endif # Build tools CC ?= gcc @@ -16,6 +21,19 @@ RM ?= rm -f MKDIR_P ?= mkdir -p PKG_CONFIG ?= pkg-config +# Optional custom libvips prefix for PLATFORM_PROVIDED_LIBVIPS mode +ifdef VIX_LIBVIPS_PREFIX + ifneq ($(filter /%,$(VIX_LIBVIPS_PREFIX)),) + VIX_LIBVIPS_PREFIX_ABS := $(VIX_LIBVIPS_PREFIX) + else + VIX_LIBVIPS_PREFIX_ABS := $(abspath $(PROJECT_ROOT)/$(VIX_LIBVIPS_PREFIX)) + endif + VIPS_PKG_CONFIG_PATH := $(VIX_LIBVIPS_PREFIX_ABS)/lib/pkgconfig$(if $(PKG_CONFIG_PATH),:$(PKG_CONFIG_PATH)) + VIPS_PKG_CONFIG := env PKG_CONFIG_PATH="$(VIPS_PKG_CONFIG_PATH)" $(PKG_CONFIG) +else + VIPS_PKG_CONFIG := $(PKG_CONFIG) +endif + # System detection and platform-specific configuration UNAME_SYS := $(shell uname -s 2>/dev/null || echo Unknown) UNAME_ARCH := $(shell uname -m 2>/dev/null || echo Unknown) @@ -40,11 +58,16 @@ LDLIBS += -L "$(ERL_INTERFACE_LIB_DIR)" LDFLAGS += -shared ifeq ($(VIX_COMPILATION_MODE), PLATFORM_PROVIDED_LIBVIPS) - VIPS_CFLAGS := $(shell ${PKG_CONFIG} --cflags vips 2>/dev/null) - VIPS_LIBS := $(shell ${PKG_CONFIG} --libs vips 2>/dev/null) + VIPS_CFLAGS := $(shell ${VIPS_PKG_CONFIG} --cflags vips 2>/dev/null) + VIPS_LIBS := $(shell ${VIPS_PKG_CONFIG} --libs vips 2>/dev/null) + VIPS_VERSION := $(shell ${VIPS_PKG_CONFIG} --modversion vips 2>/dev/null) CFLAGS += $(VIPS_CFLAGS) LDLIBS += $(VIPS_LIBS) PKG_CONFIG_CHECK = check-pkg-config + + ifdef VIX_LIBVIPS_PREFIX + LDFLAGS += -Wl,-rpath,$(VIX_LIBVIPS_PREFIX_ABS)/lib + endif else PKG_CONFIG_CHECK = PRECOMPILED_LIBVIPS_PATH = $(PREFIX)/precompiled_libvips @@ -102,6 +125,41 @@ OBJ := $(SRC:.c=.o) # Dependency files for better incremental builds DEP := $(SRC:.c=.d) +# Build configuration stamps to force rebuilds when relevant env changes +COMPILE_CONFIG = $(PREFIX)/compile.config +LINK_CONFIG = $(PREFIX)/link.config + +$(COMPILE_CONFIG): FORCE | $(PREFIX) + $(Q){ \ + printf '%s\n' \ + "CC=$(CC)" \ + "VIX_COMPILATION_MODE=$(VIX_COMPILATION_MODE)" \ + "VIX_LIBVIPS_PREFIX_ABS=$(VIX_LIBVIPS_PREFIX_ABS)" \ + "VIPS_VERSION=$(VIPS_VERSION)" \ + "CFLAGS=$(CFLAGS)"; \ + } > "$@.tmp" + $(Q)if [ ! -f "$@" ] || ! cmp -s "$@.tmp" "$@"; then \ + mv "$@.tmp" "$@"; \ + else \ + $(RM) "$@.tmp"; \ + fi + +$(LINK_CONFIG): FORCE | $(PREFIX) + $(Q){ \ + printf '%s\n' \ + "CC=$(CC)" \ + "VIX_COMPILATION_MODE=$(VIX_COMPILATION_MODE)" \ + "VIX_LIBVIPS_PREFIX_ABS=$(VIX_LIBVIPS_PREFIX_ABS)" \ + "VIPS_VERSION=$(VIPS_VERSION)" \ + "LDFLAGS=$(LDFLAGS)" \ + "LDLIBS=$(LDLIBS)"; \ + } > "$@.tmp" + $(Q)if [ ! -f "$@" ] || ! cmp -s "$@.tmp" "$@"; then \ + mv "$@.tmp" "$@"; \ + else \ + $(RM) "$@.tmp"; \ + fi + # Main targets all: check-env $(PKG_CONFIG_CHECK) $(VIX) $(SAY) "Build complete: $(VIX)" @@ -131,17 +189,36 @@ check-pkg-config: echo "Error: ${PKG_CONFIG} (PKG_CONFIG) not found but required for PLATFORM_PROVIDED_LIBVIPS"; \ exit 1; \ fi - @if ! ${PKG_CONFIG} --exists vips; then \ + @if ! ${VIPS_PKG_CONFIG} --exists vips; then \ echo "Error: vips not found via pkg-config"; \ + if [ -n "$(VIX_LIBVIPS_PREFIX_ABS)" ]; then \ + echo "Checked custom libvips prefix: $(VIX_LIBVIPS_PREFIX_ABS)"; \ + fi; \ echo "Please install libvips development headers"; \ exit 1; \ fi + @if [ -n "$(VIX_LIBVIPS_PREFIX_ABS)" ]; then \ + resolved_prefix=$$(${VIPS_PKG_CONFIG} --variable=prefix vips 2>/dev/null || true); \ + if [ -z "$$resolved_prefix" ]; then \ + echo "Error: failed to resolve the libvips prefix via pkg-config"; \ + exit 1; \ + fi; \ + if [ "$$resolved_prefix" != "$(VIX_LIBVIPS_PREFIX_ABS)" ]; then \ + if [ -d "$$resolved_prefix" ] && [ -d "$(VIX_LIBVIPS_PREFIX_ABS)" ] && [ "$$resolved_prefix" -ef "$(VIX_LIBVIPS_PREFIX_ABS)" ]; then \ + :; \ + else \ + echo "Error: libvips resolved to '$$resolved_prefix' instead of '$(VIX_LIBVIPS_PREFIX_ABS)'"; \ + echo "VIX_LIBVIPS_PREFIX must point to the libvips installation that should be used"; \ + exit 1; \ + fi; \ + fi; \ + fi # Create output directory $(PREFIX): $(Q)$(MKDIR_P) "$@" -$(OBJ): Makefile $(PRECOMPILED_LIBVIPS_PREREQUISITE) +$(OBJ): Makefile $(COMPILE_CONFIG) $(PRECOMPILED_LIBVIPS_PREREQUISITE) # Dependency generation %.d: %.c @@ -155,7 +232,7 @@ $(OBJ): Makefile $(PRECOMPILED_LIBVIPS_PREREQUISITE) $(c_verbose)$(CC) -c $(CFLAGS) -MMD -MP -o $@ $< # Final linking -$(VIX): $(PREFIX) $(OBJ) +$(VIX): $(PREFIX) $(LINK_CONFIG) $(OBJ) $(link_verbose)$(CC) $(OBJ) -o $@ $(LDFLAGS) $(LDLIBS) # Precompiled libvips setup @@ -166,7 +243,7 @@ $(PRECOMPILED_LIBVIPS_PREREQUISITE): # Clean targets clean: $(SAY) "Cleaning build artifacts..." - $(Q)$(RM) $(VIX) $(OBJ) $(DEP) + $(Q)$(RM) $(VIX) $(OBJ) $(DEP) $(COMPILE_CONFIG) $(LINK_CONFIG) clean_precompiled_libvips: clean $(SAY) "Cleaning precompiled libvips..." @@ -184,6 +261,7 @@ help: @echo "" @echo "Build configuration:" @echo " VIX_COMPILATION_MODE: $(VIX_COMPILATION_MODE)" + @echo " VIX_LIBVIPS_PREFIX: $(VIX_LIBVIPS_PREFIX_ABS)" @echo " System: $(UNAME_SYS) $(UNAME_ARCH)" @echo " CC: $(CC)" @echo "" @@ -200,9 +278,13 @@ debug: @echo "LDFLAGS: $(LDFLAGS)" @echo "LDLIBS: $(LDLIBS)" @echo "PKG_CONFIG: $(PKG_CONFIG)" + @echo "VIPS_PKG_CONFIG: $(VIPS_PKG_CONFIG)" + @echo "VIPS_VERSION: $(VIPS_VERSION)" @echo "SRC: $(SRC)" @echo "OBJ: $(OBJ)" @echo "PREFIX: $(PREFIX)" @echo "VIX: $(VIX)" -.PHONY: all clean clean_precompiled_libvips calling_from_make install check-env check-pkg-config help debug +FORCE: + +.PHONY: FORCE all clean clean_precompiled_libvips calling_from_make install check-env check-pkg-config help debug diff --git a/mix.exs b/mix.exs index ac0ae07..5532a8d 100644 --- a/mix.exs +++ b/mix.exs @@ -3,6 +3,11 @@ defmodule Vix.MixProject do @version "0.38.0" @scm_url "https://github.com/akash-akya/vix" + @valid_compilation_modes [ + "PRECOMPILED_NIF_AND_LIBVIPS", + "PRECOMPILED_LIBVIPS", + "PLATFORM_PROVIDED_LIBVIPS" + ] def project do [ @@ -124,6 +129,8 @@ defmodule Vix.MixProject do [ maintainers: ["Akash Hiremath"], licenses: ["MIT"], + # Keep the Hex package limited to runtime build inputs. + # Repository-only helpers such as scripts/ are intentionally excluded. files: ~w(lib build_scripts checksum.exs mix.exs README.md LICENSE Makefile c_src/Makefile c_src/*.{h,c} c_src/g_object/*.{h,c}), links: %{ @@ -180,7 +187,14 @@ defmodule Vix.MixProject do defp make_force_build, do: compilation_mode() == "PRECOMPILED_LIBVIPS" defp compilation_mode do - (System.get_env("VIX_COMPILATION_MODE") || "PRECOMPILED_NIF_AND_LIBVIPS") - |> String.upcase() + mode = + (System.get_env("VIX_COMPILATION_MODE") || "PRECOMPILED_NIF_AND_LIBVIPS") + |> String.upcase() + + if mode in @valid_compilation_modes do + mode + else + raise "invalid VIX_COMPILATION_MODE #{inspect(mode)}. Valid modes: #{Enum.join(@valid_compilation_modes, ", ")}" + end end end diff --git a/scripts/build_libvips.sh b/scripts/build_libvips.sh new file mode 100755 index 0000000..21e083d --- /dev/null +++ b/scripts/build_libvips.sh @@ -0,0 +1,191 @@ +#!/usr/bin/env bash +set -euo pipefail + +usage() { + cat <<'EOF' +Usage: ./scripts/build_libvips.sh [destination] [--ref ] [--overwrite] + +Download, build, and install libvips from an upstream source archive into a +private prefix. + +Arguments: + [destination] Install prefix for libvips. + Defaults to ./.libvips/. + +Options: + --ref Build a specific Git ref such as a branch, tag, or + commit SHA. Defaults to the latest upstream tagged + release. + --overwrite Replace an existing destination after a successful staged + build. + --help Show this help. +EOF +} + +die() { + printf 'Error: %s\n' "$*" >&2 + exit 1 +} + +log() { + printf '==> %s\n' "$*" +} + +latest_ref() { + local latest_url latest_tag + + latest_url=$(curl -fsSL --retry 3 --connect-timeout 30 \ + -o /dev/null -w '%{url_effective}' \ + https://github.com/libvips/libvips/releases/latest) + + latest_tag=${latest_url##*/} + + [ -n "$latest_tag" ] || die "Failed to resolve the latest libvips release" + + printf '%s\n' "$latest_tag" +} + +workdir= +cleanup() { + local status=$? + + if [ -n "${workdir:-}" ] && [ -d "$workdir" ]; then + if [ "$status" -eq 0 ]; then + rm -rf "$workdir" + else + printf 'Build failed. Temporary files were kept at %s\n' "$workdir" >&2 + fi + fi +} +trap cleanup EXIT + +destination= +ref= +overwrite=0 + +while [ "$#" -gt 0 ]; do + case "$1" in + --ref) + [ "$#" -ge 2 ] || die "--ref requires a value" + ref=$2 + shift 2 + ;; + --overwrite) + overwrite=1 + shift + ;; + --help|-h) + usage + exit 0 + ;; + --*) + die "Unknown option: $1" + ;; + *) + if [ -n "$destination" ]; then + die "Only one destination may be provided" + fi + destination=$1 + shift + ;; + esac +done + +if [ -z "$ref" ]; then + log "Resolving latest libvips release" + ref=$(latest_ref) +fi + +[ -n "$ref" ] || die "libvips ref cannot be empty" + +if [ -z "$destination" ]; then + destination="./.libvips/$ref" +fi + +if [ "$destination" = "/" ]; then + die "Refusing to install into /" +fi + +if [[ "$destination" = /* ]]; then + destination_path=$destination +else + destination_path="$PWD/$destination" +fi + +if [ "$destination_path" != "/" ]; then + destination_path=${destination_path%/} +fi + +[ -e "$destination_path" ] && [ "$overwrite" -ne 1 ] && + die "Destination already exists: $destination_path (pass --overwrite to replace it)" + +archive_name="libvips-${ref}.tar.gz" +archive_url="https://codeload.github.com/libvips/libvips/tar.gz/${ref}" + +workdir=$(mktemp -d) +archive_path="$workdir/$archive_name" +source_root="$workdir/src" +source_path="$source_root/libvips" +build_path="$workdir/build" +stage_root="$workdir/stage" +staged_destination="$stage_root$destination_path" + +mkdir -p "$source_root" +mkdir -p "$stage_root" + +log "Downloading libvips ${ref}" +curl -fL --retry 3 --connect-timeout 30 --max-time 1800 \ + "$archive_url" -o "$archive_path" + +log "Extracting source archive" +mkdir -p "$source_path" +tar -xf "$archive_path" -C "$source_path" --strip-components=1 + +log "Configuring Meson build" +meson setup "$build_path" "$source_path" \ + --prefix="$destination_path" \ + --libdir=lib \ + --buildtype=release \ + --default-library=shared \ + --backend=ninja \ + --wrap-mode=nofallback \ + -Dexamples=false \ + -Dcplusplus=false \ + -Ddocs=false \ + -Dcpp-docs=false \ + -Dintrospection=disabled \ + -Dvapi=false \ + -Dheif-module=disabled \ + -Djpeg-xl-module=disabled \ + -Dmagick-module=disabled \ + -Dopenslide-module=disabled \ + -Dpoppler-module=disabled + +log "Compiling libvips" +meson compile -C "$build_path" + +log "Installing into staged prefix" +DESTDIR="$stage_root" meson install -C "$build_path" --no-rebuild + +[ -d "$staged_destination" ] || + die "Expected staged install directory not found: $staged_destination" + +log "Publishing install prefix" +mkdir -p "$(dirname "$destination_path")" + +if [ "$overwrite" -eq 1 ] && [ -e "$destination_path" ]; then + rm -rf "$destination_path" +fi + +mv "$staged_destination" "$destination_path" + +log "libvips ${ref} installed to $destination_path" +cat <