Skip to content

Commit

Permalink
c
Browse files Browse the repository at this point in the history
  • Loading branch information
nameexhaustion committed Mar 7, 2025
1 parent 251fab5 commit b218d7e
Show file tree
Hide file tree
Showing 36 changed files with 84 additions and 107 deletions.
10 changes: 0 additions & 10 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 0 additions & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,6 @@ ndarray = { version = "0.16", default-features = false }
num-traits = "0.2"
numpy = "0.23"
object_store = { version = "0.11", default-features = false }
once_cell = "1"
parking_lot = "0.12"
percent-encoding = "2.3"
pin-project-lite = "0.2"
Expand Down
1 change: 0 additions & 1 deletion crates/polars-core/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,6 @@ indexmap = { workspace = true }
itoa = { workspace = true }
ndarray = { workspace = true, optional = true }
num-traits = { workspace = true }
once_cell = { workspace = true }
rand = { workspace = true, optional = true, features = ["small_rng", "std"] }
rand_distr = { workspace = true, optional = true }
rayon = { workspace = true }
Expand Down
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
use std::hash::{Hash, Hasher};
use std::sync::atomic::{AtomicBool, AtomicU32, Ordering};
use std::sync::{Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard};
use std::sync::{LazyLock, Mutex, RwLock, RwLockReadGuard, RwLockWriteGuard};

use hashbrown::hash_table::Entry;
use hashbrown::HashTable;
use once_cell::sync::Lazy;
use polars_utils::aliases::PlRandomState;
use polars_utils::pl_str::PlSmallStr;

Expand Down Expand Up @@ -250,4 +249,4 @@ impl StringCache {
}
}

pub(crate) static STRING_CACHE: Lazy<StringCache> = Lazy::new(Default::default);
pub(crate) static STRING_CACHE: LazyLock<StringCache> = LazyLock::new(Default::default);
6 changes: 3 additions & 3 deletions crates/polars-core/src/chunked_array/object/registry.rs
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,10 @@
use std::any::Any;
use std::fmt::{Debug, Formatter};
use std::ops::Deref;
use std::sync::{Arc, RwLock};
use std::sync::{Arc, LazyLock, RwLock};

use arrow::array::ArrayRef;
use arrow::datatypes::ArrowDataType;
use once_cell::sync::Lazy;
use polars_utils::pl_str::PlSmallStr;

use crate::chunked_array::object::builder::ObjectChunkedBuilder;
Expand All @@ -36,7 +35,8 @@ impl Debug for ObjectRegistry {
}
}

static GLOBAL_OBJECT_REGISTRY: Lazy<RwLock<Option<ObjectRegistry>>> = Lazy::new(Default::default);
static GLOBAL_OBJECT_REGISTRY: LazyLock<RwLock<Option<ObjectRegistry>>> =
LazyLock::new(Default::default);

/// This trait can be registered, after which that global registration
/// can be used to materialize object types
Expand Down
8 changes: 5 additions & 3 deletions crates/polars-core/src/chunked_array/temporal/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@ mod datetime;
mod duration;
#[cfg(feature = "dtype-time")]
mod time;
#[cfg(feature = "timezones")]
use std::sync::LazyLock;

#[cfg(feature = "dtype-date")]
use chrono::NaiveDate;
use chrono::NaiveDateTime;
Expand All @@ -16,8 +19,6 @@ use chrono::NaiveTime;
#[cfg(feature = "timezones")]
use chrono_tz::Tz;
#[cfg(feature = "timezones")]
use once_cell::sync::Lazy;
#[cfg(feature = "timezones")]
use polars_utils::pl_str::PlSmallStr;
#[cfg(all(feature = "regex", feature = "timezones"))]
use regex::Regex;
Expand All @@ -38,7 +39,8 @@ static FIXED_OFFSET_PATTERN: &str = r#"(?x)
$
"#;
#[cfg(feature = "timezones")]
static FIXED_OFFSET_RE: Lazy<Regex> = Lazy::new(|| Regex::new(FIXED_OFFSET_PATTERN).unwrap());
static FIXED_OFFSET_RE: LazyLock<Regex> =
LazyLock::new(|| Regex::new(FIXED_OFFSET_PATTERN).unwrap());

#[cfg(feature = "timezones")]
pub fn validate_time_zone(tz: &str) -> PolarsResult<()> {
Expand Down
13 changes: 6 additions & 7 deletions crates/polars-core/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,17 @@ pub mod testing;
#[cfg(test)]
mod tests;

use std::sync::Mutex;
use std::sync::{LazyLock, Mutex};
use std::time::{SystemTime, UNIX_EPOCH};

pub use datatypes::SchemaExtPl;
pub use hashing::IdBuildHasher;
use once_cell::sync::Lazy;
use rayon::{ThreadPool, ThreadPoolBuilder};

#[cfg(feature = "dtype-categorical")]
pub use crate::chunked_array::logical::categorical::string_cache::*;

pub static PROCESS_ID: Lazy<u128> = Lazy::new(|| {
pub static PROCESS_ID: LazyLock<u128> = LazyLock::new(|| {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap()
Expand All @@ -47,7 +46,7 @@ pub static PROCESS_ID: Lazy<u128> = Lazy::new(|| {

// this is re-exported in utils for polars child crates
#[cfg(not(target_family = "wasm"))] // only use this on non wasm targets
pub static POOL: Lazy<ThreadPool> = Lazy::new(|| {
pub static POOL: LazyLock<ThreadPool> = LazyLock::new(|| {
let thread_name = std::env::var("POLARS_THREAD_NAME").unwrap_or_else(|_| "polars".to_string());
ThreadPoolBuilder::new()
.num_threads(
Expand All @@ -65,7 +64,7 @@ pub static POOL: Lazy<ThreadPool> = Lazy::new(|| {
});

#[cfg(all(target_os = "emscripten", target_family = "wasm"))] // Use 1 rayon thread on emscripten
pub static POOL: Lazy<ThreadPool> = Lazy::new(|| {
pub static POOL: LazyLock<ThreadPool> = LazyLock::new(|| {
ThreadPoolBuilder::new()
.num_threads(1)
.use_current_thread()
Expand All @@ -74,10 +73,10 @@ pub static POOL: Lazy<ThreadPool> = Lazy::new(|| {
});

#[cfg(all(not(target_os = "emscripten"), target_family = "wasm"))] // use this on other wasm targets
pub static POOL: Lazy<polars_utils::wasm::Pool> = Lazy::new(|| polars_utils::wasm::Pool);
pub static POOL: LazyLock<polars_utils::wasm::Pool> = LazyLock::new(|| polars_utils::wasm::Pool);

// utility for the tests to ensure a single thread can execute
pub static SINGLE_LOCK: Lazy<Mutex<()>> = Lazy::new(|| Mutex::new(()));
pub static SINGLE_LOCK: LazyLock<Mutex<()>> = LazyLock::new(|| Mutex::new(()));

/// Default length for a `.head()` call
pub(crate) const HEAD_DEFAULT_LENGTH: usize = 10;
Expand Down
7 changes: 3 additions & 4 deletions crates/polars-core/src/random.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
use std::sync::Mutex;
use std::sync::{LazyLock, Mutex};

use once_cell::sync::Lazy;
use rand::prelude::*;

static POLARS_GLOBAL_RNG_STATE: Lazy<Mutex<SmallRng>> =
Lazy::new(|| Mutex::new(SmallRng::from_entropy()));
static POLARS_GLOBAL_RNG_STATE: LazyLock<Mutex<SmallRng>> =
LazyLock::new(|| Mutex::new(SmallRng::from_entropy()));

pub(crate) fn get_global_random_u64() -> u64 {
POLARS_GLOBAL_RNG_STATE.lock().unwrap().next_u64()
Expand Down
1 change: 0 additions & 1 deletion crates/polars-expr/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@ arrow = { workspace = true }
bitflags = { workspace = true }
hashbrown = { workspace = true }
num-traits = { workspace = true }
once_cell = { workspace = true }
polars-compute = { workspace = true }
polars-core = { workspace = true, features = ["lazy", "zip_with", "random"] }
polars-io = { workspace = true, features = ["lazy"] }
Expand Down
7 changes: 3 additions & 4 deletions crates/polars-expr/src/state/execution_state.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
use std::borrow::Cow;
use std::sync::atomic::{AtomicBool, AtomicI64, AtomicU8, Ordering};
use std::sync::{Mutex, RwLock};
use std::sync::{Mutex, OnceLock, RwLock};
use std::time::Duration;

use bitflags::bitflags;
use once_cell::sync::OnceCell;
use polars_core::config::verbose;
use polars_core::prelude::*;
use polars_ops::prelude::ChunkJoinOptIds;
Expand Down Expand Up @@ -101,7 +100,7 @@ impl From<u8> for StateFlags {
}
}

type CachedValue = Arc<(AtomicI64, OnceCell<DataFrame>)>;
type CachedValue = Arc<(AtomicI64, OnceLock<DataFrame>)>;

/// State/ cache that is maintained during the Execution of the physical plan.
pub struct ExecutionState {
Expand Down Expand Up @@ -229,7 +228,7 @@ impl ExecutionState {
guard
.entry(key)
.or_insert_with(|| {
Arc::new((AtomicI64::new(cache_hits as i64), OnceCell::new()))
Arc::new((AtomicI64::new(cache_hits as i64), OnceLock::new()))
})
.clone()
},
Expand Down
1 change: 0 additions & 1 deletion crates/polars-io/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@ memchr = { workspace = true }
memmap = { workspace = true }
num-traits = { workspace = true }
object_store = { workspace = true, optional = true }
once_cell = { workspace = true }
percent-encoding = { workspace = true }
pyo3 = { workspace = true, optional = true }
rayon = { workspace = true }
Expand Down
7 changes: 3 additions & 4 deletions crates/polars-io/src/cloud/object_store_setup.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
use std::sync::Arc;
use std::sync::{Arc, LazyLock};

use object_store::local::LocalFileSystem;
use object_store::ObjectStore;
use once_cell::sync::Lazy;
use polars_core::config::{self, verbose_print_sensitive};
use polars_error::{polars_bail, to_compute_err, PolarsError, PolarsResult};
use polars_utils::aliases::PlHashMap;
Expand All @@ -18,8 +17,8 @@ use crate::cloud::CloudConfig;
/// get rate limited when querying the DNS (can take up to 5s).
/// Other reasons are connection pools that must be shared between as much as possible.
#[allow(clippy::type_complexity)]
static OBJECT_STORE_CACHE: Lazy<RwLock<PlHashMap<Vec<u8>, PolarsObjectStore>>> =
Lazy::new(Default::default);
static OBJECT_STORE_CACHE: LazyLock<RwLock<PlHashMap<Vec<u8>, PolarsObjectStore>>> =
LazyLock::new(Default::default);

#[allow(dead_code)]
fn err_missing_feature(feature: &str, scheme: &str) -> PolarsResult<Arc<dyn ObjectStore>> {
Expand Down
8 changes: 4 additions & 4 deletions crates/polars-io/src/cloud/options.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ use std::io::Read;
#[cfg(feature = "aws")]
use std::path::Path;
use std::str::FromStr;
use std::sync::LazyLock;

#[cfg(feature = "aws")]
use object_store::aws::AmazonS3Builder;
Expand All @@ -20,7 +21,6 @@ pub use object_store::gcp::GoogleConfigKey;
use object_store::ClientOptions;
#[cfg(any(feature = "aws", feature = "gcp", feature = "azure"))]
use object_store::{BackoffConfig, RetryConfig};
use once_cell::sync::Lazy;
use polars_error::*;
#[cfg(feature = "aws")]
use polars_utils::cache::FastFixedCache;
Expand All @@ -41,11 +41,11 @@ use crate::file_cache::get_env_file_cache_ttl;
use crate::pl_async::with_concurrency_budget;

#[cfg(feature = "aws")]
static BUCKET_REGION: Lazy<
static BUCKET_REGION: LazyLock<
std::sync::Mutex<
FastFixedCache<polars_utils::pl_str::PlSmallStr, polars_utils::pl_str::PlSmallStr>,
>,
> = Lazy::new(|| std::sync::Mutex::new(FastFixedCache::new(32)));
> = LazyLock::new(|| std::sync::Mutex::new(FastFixedCache::new(32)));

/// The type of the config keys must satisfy the following requirements:
/// 1. must be easily collected into a HashMap, the type required by the object_crate API.
Expand Down Expand Up @@ -91,7 +91,7 @@ impl Default for CloudOptions {

impl CloudOptions {
pub fn default_static_ref() -> &'static Self {
static DEFAULT: Lazy<CloudOptions> = Lazy::new(|| CloudOptions {
static DEFAULT: LazyLock<CloudOptions> = LazyLock::new(|| CloudOptions {
max_retries: 2,
#[cfg(feature = "file_cache")]
file_cache_ttl: get_env_file_cache_ttl(),
Expand Down
5 changes: 2 additions & 3 deletions crates/polars-io/src/file_cache/cache.rs
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
use std::path::Path;
use std::sync::atomic::AtomicU64;
use std::sync::{Arc, RwLock};
use std::sync::{Arc, LazyLock, RwLock};

use once_cell::sync::Lazy;
use polars_core::config;
use polars_error::PolarsResult;
use polars_utils::aliases::PlHashMap;
Expand All @@ -13,7 +12,7 @@ use super::file_fetcher::FileFetcher;
use super::utils::FILE_CACHE_PREFIX;
use crate::path_utils::{ensure_directory_init, is_cloud_url};

pub static FILE_CACHE: Lazy<FileCache> = Lazy::new(|| {
pub static FILE_CACHE: LazyLock<FileCache> = LazyLock::new(|| {
let prefix = FILE_CACHE_PREFIX.as_ref();
let prefix = Arc::<Path>::from(prefix);

Expand Down
5 changes: 2 additions & 3 deletions crates/polars-io/src/file_cache/cache_lock.rs
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
use std::sync::atomic::AtomicBool;
use std::sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard};
use std::sync::{Arc, LazyLock, RwLock, RwLockReadGuard, RwLockWriteGuard};
use std::time::Duration;

use fs4::fs_std::FileExt;
use once_cell::sync::Lazy;

use super::utils::FILE_CACHE_PREFIX;
use crate::pl_async;

pub(super) static GLOBAL_FILE_CACHE_LOCK: Lazy<GlobalLock> = Lazy::new(|| {
pub(super) static GLOBAL_FILE_CACHE_LOCK: LazyLock<GlobalLock> = LazyLock::new(|| {
let path = FILE_CACHE_PREFIX.join(".process-lock");

let file = std::fs::OpenOptions::new()
Expand Down
5 changes: 2 additions & 3 deletions crates/polars-io/src/file_cache/entry.rs
Original file line number Diff line number Diff line change
@@ -1,10 +1,9 @@
use std::io::{Seek, SeekFrom};
use std::path::{Path, PathBuf};
use std::sync::atomic::AtomicU64;
use std::sync::{Arc, Mutex};
use std::sync::{Arc, LazyLock, Mutex};

use fs4::fs_std::FileExt;
use once_cell::sync::Lazy;
use polars_core::config;
use polars_error::{polars_bail, to_compute_err, PolarsError, PolarsResult};

Expand Down Expand Up @@ -156,7 +155,7 @@ impl Inner {
// * Some(true) => always raise
// * Some(false) => never raise
// * None => do not raise if fallocate() is not permitted, otherwise raise.
static RAISE_ALLOC_ERROR: Lazy<Option<bool>> = Lazy::new(|| {
static RAISE_ALLOC_ERROR: LazyLock<Option<bool>> = LazyLock::new(|| {
let v = match std::env::var("POLARS_IGNORE_FILE_CACHE_ALLOCATE_ERROR").as_deref() {
Ok("1") => Some(false),
Ok("0") => Some(true),
Expand Down
Loading

0 comments on commit b218d7e

Please sign in to comment.