"""Tests for malformed binary files."""
HDR_OFF_SAMPLES = 28
- HDR_OFF_THREADS = 32
- HDR_OFF_STR_TABLE = 36
- HDR_OFF_FRAME_TABLE = 44
+ HDR_OFF_THREADS = 36
+ HDR_OFF_STR_TABLE = 40
+ HDR_OFF_FRAME_TABLE = 48
FILE_HEADER_PLACEHOLDER_SIZE = 64
FILE_FOOTER_SIZE = 32
FTR_OFF_STRINGS = 0
with open(filename, "r+b") as raw:
raw.seek(self.HDR_OFF_SAMPLES)
- raw.write(struct.pack("=I", 2))
+ raw.write(struct.pack("=Q", 2))
with BinaryReader(filename) as reader:
self.assertEqual(reader.get_info()["sample_count"], 2)
f"possible {max_frames}",
)
+ def test_sample_count_reads_full_64_bits(self):
+ """sample_count values requiring the upper 32 bits decode correctly."""
+ filename = self.create_binary_file([], compression="none")
+ big_count = 0x1_0002_0003
+
+ with open(filename, "r+b") as raw:
+ raw.seek(self.HDR_OFF_SAMPLES)
+ raw.write(struct.pack("=Q", big_count))
+
+ with BinaryReader(filename) as reader:
+ self.assertEqual(reader.get_info()["sample_count"], big_count)
+
+ def test_sample_count_boundary_values(self):
+ """Values above the old u32 ceiling decode fine."""
+ filename = self.create_binary_file([], compression="none")
+
+ for value in (0xFFFFFFFF - 1, 0xFFFFFFFF, 0xFFFFFFFF + 1):
+ with self.subTest(value=value):
+ with open(filename, "r+b") as raw:
+ raw.seek(self.HDR_OFF_SAMPLES)
+ raw.write(struct.pack("=Q", value))
+
+ with BinaryReader(filename) as reader:
+ self.assertEqual(reader.get_info()["sample_count"], value)
+
class TestBinaryEncodings(BinaryFormatTestBase):
"""Tests specifically targeting different stack encodings."""
--- /dev/null
+Store the sample count in :mod:`profiling.sampling` binary format, as a
+64-bit integer, widening from previously used 32-bit integer. This breaks
+the existing recordings. Long or thread-heavy profiling sessions will no
+longer fail because of :exc:`OverflowError`. Patch by Maurycy
+Pawłowski-Wieroński.
#define HDR_OFF_INTERVAL (HDR_OFF_START_TIME + HDR_SIZE_START_TIME)
#define HDR_SIZE_INTERVAL 8
#define HDR_OFF_SAMPLES (HDR_OFF_INTERVAL + HDR_SIZE_INTERVAL)
-#define HDR_SIZE_SAMPLES 4
+#define HDR_SIZE_SAMPLES 8
#define HDR_OFF_THREADS (HDR_OFF_SAMPLES + HDR_SIZE_SAMPLES)
#define HDR_SIZE_THREADS 4
#define HDR_OFF_STR_TABLE (HDR_OFF_THREADS + HDR_SIZE_THREADS)
/* Metadata */
uint64_t start_time_us;
uint64_t sample_interval_us;
- uint32_t total_samples;
+ uint64_t total_samples;
/* String hash table: PyObject* -> uint32_t index */
_Py_hashtable_t *string_hash;
int needs_swap; /* Non-zero if file was written on different-endian system */
uint64_t start_time_us;
uint64_t sample_interval_us;
- uint32_t sample_count;
+ uint64_t sample_count;
uint32_t thread_count;
uint64_t string_table_offset;
uint64_t frame_table_offset;
/* Read header fields with byte-swapping if needed */
uint64_t start_time_us, sample_interval_us, string_table_offset, frame_table_offset;
- uint32_t sample_count, thread_count, compression_type;
+ uint64_t sample_count;
+ uint32_t thread_count, compression_type;
memcpy(&start_time_us, &data[HDR_OFF_START_TIME], HDR_SIZE_START_TIME);
memcpy(&sample_interval_us, &data[HDR_OFF_INTERVAL], HDR_SIZE_INTERVAL);
reader->start_time_us = SWAP64_IF(reader->needs_swap, start_time_us);
reader->sample_interval_us = SWAP64_IF(reader->needs_swap, sample_interval_us);
- reader->sample_count = SWAP32_IF(reader->needs_swap, sample_count);
+ reader->sample_count = SWAP64_IF(reader->needs_swap, sample_count);
reader->thread_count = SWAP32_IF(reader->needs_swap, thread_count);
reader->string_table_offset = SWAP64_IF(reader->needs_swap, string_table_offset);
reader->frame_table_offset = SWAP64_IF(reader->needs_swap, frame_table_offset);
/* Helper to invoke progress callback, returns -1 on error */
static inline int
-invoke_progress_callback(PyObject *callback, Py_ssize_t current, uint32_t total)
+invoke_progress_callback(PyObject *callback, Py_ssize_t current, uint64_t total)
{
if (callback && callback != Py_None) {
- PyObject *result = PyObject_CallFunction(callback, "nI", current, total);
+ PyObject *result = PyObject_CallFunction(callback, "nK", current, (unsigned long long)total);
if (result) {
Py_DECREF(result);
} else {
if ((uint64_t)replayed != reader->sample_count) {
PyErr_Format(PyExc_ValueError,
- "Sample count mismatch: header declares %u samples but replay decoded %zd",
- reader->sample_count, replayed);
+ "Sample count mismatch: header declares %llu samples but replay decoded %zd",
+ (unsigned long long)reader->sample_count, replayed);
return -1;
}
return NULL;
}
return Py_BuildValue(
- "{s:I, s:N, s:K, s:K, s:I, s:I, s:I, s:I, s:i}",
+ "{s:I, s:N, s:K, s:K, s:K, s:I, s:I, s:I, s:i}",
"version", BINARY_FORMAT_VERSION,
"python_version", py_version,
"start_time_us", reader->start_time_us,
process_thread_sample(BinaryWriter *writer, PyObject *thread_info,
uint32_t interpreter_id, uint64_t timestamp_us)
{
- if (writer->total_samples >= UINT32_MAX) {
- PyErr_SetString(PyExc_OverflowError,
- "too many samples for binary format");
- return -1;
- }
-
PyObject *thread_id_obj = PyStructSequence_GET_ITEM(thread_info, 0);
PyObject *status_obj = PyStructSequence_GET_ITEM(thread_info, 1);
PyObject *frame_list = PyStructSequence_GET_ITEM(thread_info, 2);
typedef struct {
PyObject_HEAD
BinaryWriter *writer;
- uint32_t cached_total_samples; /* Preserved after finalize */
+ uint64_t cached_total_samples; /* Preserved after finalize */
} BinaryWriterObject;
typedef struct {
BinaryWriterObject *self = BinaryWriter_CAST(op);
if (!self->writer) {
/* Use cached value after finalize/close */
- return PyLong_FromUnsignedLong(self->cached_total_samples);
+ return PyLong_FromUnsignedLongLong(self->cached_total_samples);
}
- return PyLong_FromUnsignedLong(self->writer->total_samples);
+ return PyLong_FromUnsignedLongLong(self->writer->total_samples);
}
static PyGetSetDef BinaryWriter_getset[] = {
if (!self->reader) {
return PyLong_FromLong(0);
}
- return PyLong_FromUnsignedLong(self->reader->sample_count);
+ return PyLong_FromUnsignedLongLong(self->reader->sample_count);
}
static PyObject *
projects / thirdparty / Python / cpython.git / commitdiff
