Repository navigation
Expand file tree
/
Copy pathsetup.py
More file actions
239 lines (202 loc) · 9.04 KB
/
Copy pathsetup.py
File metadata and controls
239 lines (202 loc) · 9.04 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
import os
import platform
import sys
import sysconfig
from setuptools import Extension
from setuptools import find_packages
from setuptools import setup
from setuptools.command.build_ext import build_ext as _build_ext
LIBRARY = "orso"
def is_mac(): # pragma: no cover
return platform.system().lower() == "darwin"
# Windows is not supported from v2 on (DESIGN.md §11, decided 2026-10-16): the native
# layer is built on draken headers that need __int128 and <execinfo.h>, neither of
# which MSVC has, and the alternatives were not worth carrying.
if platform.system().lower() == "windows": # pragma: no cover
raise SystemExit("orso does not build on Windows; Linux and macOS wheels are published.")
if is_mac(): # pragma: no cover
COMPILE_FLAGS = ["-O2"]
LINK_FLAGS = ["-O2"]
else: # pragma: no cover
# NOTE: do not add -march=native here. Wheels are built on CI and published, so
# baking in the build machine's instruction set makes them crash with SIGILL on
# any older CPU. Opt in explicitly for local/self-hosted builds instead.
COMPILE_FLAGS = ["-O2"]
LINK_FLAGS = ["-O2"]
if os.environ.get("ORSO_NATIVE_ARCH") == "1":
COMPILE_FLAGS.append("-march=native")
LINK_FLAGS.append("-march=native")
__version__ = "notset"
with open(f"{LIBRARY}/version.py", mode="r") as v:
vers = v.read()
exec(vers) # nosec
with open("README.md", mode="r", encoding="UTF8") as rm:
long_description = rm.read()
def _read_requirements() -> list:
"""
Read the install requirements.
requirements.txt is the source of truth and is shipped in the sdist. The
egg-info fallback exists for trees where it is missing, but that file also
carries `[extras]` sections which are not valid requirement specifiers - reading
them verbatim made the sdist impossible to build, so stop at the first section.
"""
try:
with open("requirements.txt", "r") as f:
lines = f.read().splitlines()
except OSError:
with open(f"{LIBRARY}.egg-info/requires.txt", "r") as f:
lines = []
for line in f.read().splitlines():
if line.startswith("["): # start of an extras section
break
lines.append(line)
return [line.strip() for line in lines if line.strip() and not line.startswith("#")]
required = _read_requirements()
ext_modules = []
# ---------------------------------------------------------------------------------
# v2 native layer: one nanobind extension, `orso._native`, over the draken headers
# vendored into native/ by `make sync-draken` (DESIGN.md §3, §7).
#
# Flags follow opteryx-core's build_common.py: -O3, C++20, an ISA floor rather than
# -march=native so published wheels run on any CPU of the generation, and
# NB_FREE_THREADED when the build interpreter has the GIL disabled.
# ---------------------------------------------------------------------------------
NATIVE_DIR = "native"
DRAKEN_SOURCE_RECORD = os.path.join(NATIVE_DIR, "draken", "SOURCE")
def _draken_source_commit() -> str:
try:
with open(DRAKEN_SOURCE_RECORD) as f:
for line in f:
if line.startswith("commit: "):
return line.split(": ", 1)[1].strip()
except OSError:
pass
return "unknown"
def _native_arch_flags() -> list:
"""ISA floor per target. A universal2 macOS build (both archs in ARCHFLAGS)
gets none, because one -march cannot serve two architectures."""
archflags = os.environ.get("ARCHFLAGS", "").lower()
has_x86 = "x86_64" in archflags
has_arm = "arm64" in archflags or "aarch64" in archflags
if has_x86 and has_arm:
return []
machine = platform.machine().lower()
if has_x86 or (not has_arm and machine in ("x86_64", "amd64")):
# haswell: AVX2/BMI/FMA. Everything since 2013 on the server side.
return ["-march=haswell", "-mtune=generic"]
if has_arm or "aarch64" in machine or "arm64" in machine:
return ["-march=armv8-a+aes", "-mtune=generic"]
return []
def _native_extension():
if not os.path.exists(DRAKEN_SOURCE_RECORD):
raise SystemExit(
f"{DRAKEN_SOURCE_RECORD} not found: the vendored draken headers are missing. "
"Run `make sync-draken` (needs an opteryx-core checkout beside this one)."
)
import nanobind
nanobind_root = os.path.dirname(nanobind.__file__)
ryu_dir = os.path.join(NATIVE_DIR, "third_party", "ulfjack", "ryu")
cpp_flags = [
"-O3",
"-std=c++20",
"-fvisibility=hidden",
"-fno-strict-aliasing",
] + _native_arch_flags()
define_macros = [
("NB_COMPACT_ASSERTIONS", "1"),
# Keep orso's nanobind type registry separate from any other nanobind
# extension in the process (opteryx-core ships its own vendored nanobind).
("NB_DOMAIN", "orso"),
("ORSO_DRAKEN_SOURCE", f'"{_draken_source_commit()}"'),
]
if sysconfig.get_config_var("Py_GIL_DISABLED"):
# nanobind must see this in nb_combined.cpp AND every unit that includes its
# headers, so it is a define on the extension, not a flag on one source.
define_macros.append(("NB_FREE_THREADED", "1"))
return Extension(
name="orso._native",
sources=[
os.path.join(NATIVE_DIR, "native.cpp"),
os.path.join(NATIVE_DIR, "draken", "core", "mem_account.cpp"),
os.path.join(ryu_dir, "d2s.c"),
os.path.join(ryu_dir, "d2fixed.c"),
os.path.join(ryu_dir, "f2s.c"),
os.path.join(nanobind.source_dir(), "nb_combined.cpp"),
],
include_dirs=[
NATIVE_DIR,
os.path.join(
NATIVE_DIR, "draken"
), # "core/buffers.h", "logical_type.h", "ops/vec_result.h"
os.path.join(
NATIVE_DIR, "draken", "core"
), # draken's own quote-includes ("string_slot.h")
os.path.join(NATIVE_DIR, "third_party", "cyan4973"), # "xxhash.h"
ryu_dir, # "ryu.h"
nanobind.include_dir(),
nanobind.source_dir(),
os.path.join(nanobind_root, "ext", "robin_map", "include"),
],
define_macros=define_macros,
extra_compile_args=cpp_flags,
extra_link_args=LINK_FLAGS,
# the draken bridge resolves its symbols with dlopen/dlsym (native/draken_bridge.h);
# glibc before 2.34 (manylinux_2_28) keeps them in libdl
libraries=["dl"] if sys.platform.startswith("linux") else [],
language="c++",
depends=[
os.path.join(NATIVE_DIR, "record.h"),
os.path.join(NATIVE_DIR, "row.h"),
os.path.join(NATIVE_DIR, "store.h"),
os.path.join(NATIVE_DIR, "width.h"),
os.path.join(NATIVE_DIR, "decimal.h"),
os.path.join(NATIVE_DIR, "arrow_import.h"),
os.path.join(NATIVE_DIR, "arrow_export.h"),
os.path.join(NATIVE_DIR, "schema_slots.h"),
os.path.join(NATIVE_DIR, "serialize.h"),
os.path.join(NATIVE_DIR, "render.h"),
os.path.join(NATIVE_DIR, "columns.h"),
os.path.join(NATIVE_DIR, "draken_bridge.h"),
DRAKEN_SOURCE_RECORD,
],
)
class build_ext(_build_ext):
"""The native extension mixes C++20 units with ryu's C sources. setuptools applies
one set of extra_compile_args to every source, and clang refuses -std=c++20 for a
C unit, so give C units a C standard instead (the same fix build_common.py makes)."""
def build_extensions(self):
original_compile = self.compiler._compile
def _compile(obj, src, ext, cc_args, extra_postargs, pp_opts):
if src.endswith(".c"):
extra_postargs = [a for a in extra_postargs if not a.startswith("-std=")]
extra_postargs = [a for a in extra_postargs if a != "-fvisibility=hidden"] + [
"-std=c11"
]
return original_compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
if hasattr(self.compiler, "_compile"):
self.compiler._compile = _compile
super().build_extensions()
ext_modules.append(_native_extension())
setup_config = {
"name": LIBRARY,
"version": __version__,
"description": "🐻 DataFrame Library",
"long_description": long_description,
"long_description_content_type": "text/markdown",
"author_email": "justin.joyce@joocer.com",
"packages": find_packages(include=[LIBRARY, f"{LIBRARY}.*"]),
"url": "https://github.com/mabel-dev/orso/",
"ext_modules": ext_modules,
"cmdclass": {"build_ext": build_ext},
"install_requires": required,
# 3.9 is EOL (Oct 2025) and our own dependencies no longer publish cp39
# wheels; without this pip would offer 3.9 users an sdist they cannot use.
"python_requires": ">=3.10",
"extras_require": {
# pysimdjson provides the `simdjson` Python module used as an optional accelerator.
"simdjson": ["pysimdjson"],
# collect() and the virtual columns' materialize() return numpy arrays
"numpy": ["numpy"],
},
}
setup(**setup_config)