Loading...
Searching...
No Matches
file.hpp
Go to the documentation of this file.
1// SPDX-License-Identifier: Apache-2.0
2#pragma once
3#include "meshkit/io.hpp"
4#include "operation.hpp"
5#include <array>
6#include <atomic>
7#include <cstdio>
8#include <cerrno>
9#include <cstring>
10#include <random>
11#ifdef _WIN32
12#ifndef NOMINMAX
13#define NOMINMAX
14#endif
15#include <windows.h>
16#include <io.h>
17#else
18#include <fcntl.h>
19#include <sys/stat.h>
20#include <unistd.h>
21#endif
23{
29inline bool validPath(const std::filesystem::path &p)
30{
31 return !p.empty() && p.native().find(std::filesystem::path::value_type{}) ==
32 std::filesystem::path::string_type::npos;
33}
34
35constexpr std::uint64_t hashSeed = 14695981039346656037ull;
42inline void hashBytes(std::uint64_t &hash, const unsigned char *p, std::size_t n)
43{
44 for (std::size_t i = 0; i < n; ++i) {
45 hash ^= p[i];
46 hash *= 1099511628211ull;
47 }
48}
49
56{
57 std::filesystem::path target_, temp_;
58 bool overwrite_, durable_;
59 std::FILE *file_ = nullptr;
60
61public:
73 Transaction(const std::filesystem::path &target, const IOOptions &options)
74 : target_(target), overwrite_(options.overwrite), durable_(options.durable)
75 {
76 if (!validPath(target) || target.filename().empty() || target.filename() == "." ||
77 target.filename() == "..")
78 fail("path", "Invalid output path.", RunStatus::Failed);
79 auto parent = target.parent_path();
80 if (parent.empty())
81 parent = ".";
82 static std::atomic<std::uint64_t> sequence{0};
83 std::random_device random;
84 for (int attempt = 0; attempt < 32; ++attempt) {
85 const auto nonce = (std::uint64_t(random()) << 32) ^ random() ^ ++sequence;
86 temp_ = parent / (".meshkit-" + std::to_string(nonce) + ".tmp");
87#ifdef _WIN32
88 file_ = _wfopen(temp_.c_str(), L"wbx");
89#else
90 // Restrict access from creation, even with a permissive process umask.
91 const int fd = ::open(temp_.c_str(), O_WRONLY | O_CREAT | O_EXCL, S_IRUSR | S_IWUSR);
92 if (fd >= 0) {
93 file_ = ::fdopen(fd, "wb");
94 if (!file_) {
95 const int error = errno;
96 ::close(fd);
97 ::unlink(temp_.c_str());
98 errno = error;
99 break;
100 }
101 }
102#endif
103 if (file_)
104 return;
105 if (errno != EEXIST)
106 break;
107 }
108 fail("open_output",
109 "Cannot create an exclusive output temporary: " + std::string(std::strerror(errno)),
111 }
112
114 {
115 if (file_)
116 std::fclose(file_);
117 if (!temp_.empty()) {
118 std::error_code error;
119 std::filesystem::remove(temp_, error);
120 }
121 }
122 Transaction(const Transaction &) = delete;
124 std::FILE *file() const { return file_; }
135 void commit()
136 {
137 if (std::fflush(file_) != 0)
138 fail("write", "Cannot flush output.", RunStatus::Failed);
139 if (durable_) {
140#ifdef _WIN32
141 if (_commit(_fileno(file_)) != 0)
142 fail("sync", "Cannot synchronize output.", RunStatus::Failed);
143#else
144 if (::fsync(fileno(file_)) != 0)
145 fail("sync", "Cannot synchronize output.", RunStatus::Failed);
146#endif
147 }
148 const int closed = std::fclose(file_);
149 file_ = nullptr;
150 if (closed != 0)
151 fail("close", "Cannot close output.", RunStatus::Failed);
152#ifdef _WIN32
153 const DWORD flags =
154 (overwrite_ ? MOVEFILE_REPLACE_EXISTING : 0) | (durable_ ? MOVEFILE_WRITE_THROUGH : 0);
155 if (!MoveFileExW(temp_.c_str(), target_.c_str(), flags))
156 fail("publish", "Cannot atomically publish output (destination may already exist).",
158#else
159 if (overwrite_) {
160 if (::rename(temp_.c_str(), target_.c_str()) != 0)
161 fail("publish",
162 "Cannot atomically replace output: " + std::string(std::strerror(errno)),
164 } else {
165 if (::link(temp_.c_str(), target_.c_str()) != 0)
166 fail("publish",
167 "Cannot publish output without replacing an existing file: " +
168 std::string(std::strerror(errno)),
170 ::unlink(temp_.c_str());
171 }
172#endif
173 temp_.clear();
174 }
175};
176
183{
184 std::FILE *file_;
185 Work &work_;
186 std::array<unsigned char, 65536> buffer_{};
187 std::size_t used_ = 0;
188
189public:
191 std::uint64_t bytes = 0, hash = hashSeed;
199 Output(std::FILE *file, Work &work) : file_(file), work_(work) {}
204 void flush()
205 {
206 work_.check();
207 if (used_ && std::fwrite(buffer_.data(), 1, used_, file_) != used_)
208 fail("write", "Output write failed.", RunStatus::Failed);
209 used_ = 0;
210 }
211
218 void raw(const void *data, std::size_t n, bool hashed = true)
219 {
220 auto p = static_cast<const unsigned char *>(data);
221 if (hashed)
222 hashBytes(hash, p, n);
223 bytes += n;
224 while (n) {
225 const auto count = std::min(n, buffer_.size() - used_);
226 std::memcpy(buffer_.data() + used_, p, count);
227 used_ += count;
228 p += count;
229 n -= count;
230 if (used_ == buffer_.size())
231 flush();
232 }
233 }
234
240 void number(std::uint64_t value, int width, bool hashed = true)
241 {
242 unsigned char data[8];
243 for (int i = 0; i < width; ++i) {
244 data[i] = value & 255;
245 value >>= 8;
246 }
247 raw(data, width, hashed);
248 }
249
253 void real(double value)
254 {
255 std::uint64_t bits;
256 static_assert(sizeof(value) == sizeof(bits));
257 std::memcpy(&bits, &value, 8);
258 number(bits, 8);
259 }
260
264 void text(const std::string &s)
265 {
266 number(s.size(), 8);
267 raw(s.data(), s.size());
268 }
269};
270
275class Input
276{
277 std::istream &stream_;
278 Work &work_;
279 std::array<unsigned char, 65536> buffer_{};
280 std::size_t at_ = 0, end_ = 0;
281
282public:
284 std::uint64_t bytes = 0, hash = hashSeed;
293 Input(std::istream &stream, Work &work) : stream_(stream), work_(work) {}
301 bool byte(unsigned char &value, bool hashed = true)
302 {
303 if (at_ == end_) {
304 work_.check();
305 stream_.read(reinterpret_cast<char *>(buffer_.data()), buffer_.size());
306 end_ = std::size_t(stream_.gcount());
307 at_ = 0;
308 if (!end_) {
309 if (stream_.bad())
310 fail("read", "Input read failed.", RunStatus::Failed);
311 return false;
312 }
313 }
314 value = buffer_[at_++];
315 ++bytes;
316 if (hashed)
317 hashBytes(hash, &value, 1);
318 return true;
319 }
320
327 void raw(void *target, std::size_t n, bool hashed = true)
328 {
329 auto p = static_cast<unsigned char *>(target);
330 for (std::size_t i = 0; i < n; ++i)
331 if (!byte(p[i], hashed))
332 fail("truncated", "Truncated input.", RunStatus::Failed);
333 }
334
341 std::uint64_t number(int width, bool hashed = true)
342 {
343 unsigned char data[8];
344 raw(data, width, hashed);
345 std::uint64_t value = 0;
346 for (int i = 0; i < width; ++i)
347 value |= std::uint64_t(data[i]) << (8 * i);
348 return value;
349 }
350
355 double real()
356 {
357 const auto bits = number(8);
358 double value;
359 std::memcpy(&value, &bits, 8);
360 return value;
361 }
362
369 std::string text(std::uint64_t &remaining)
370 {
371 auto n = number(8);
372 if (n > remaining)
373 fail("metadata_limit", "Metadata size exceeds the limit.", RunStatus::ResourceLimit);
374 remaining -= n;
375 std::string result(std::size_t(n), '\0');
376 raw(result.data(), result.size());
377 return result;
378 }
379};
380
389inline void appendId(std::vector<std::int64_t> &values, std::int64_t id, std::size_t index)
390{
391 if (values.empty() && id == std::int64_t(index + 1))
392 return;
393 if (values.empty()) {
394 values.reserve(index + 1);
395 for (std::size_t i = 0; i < index; ++i)
396 values.push_back(i + 1);
397 }
398 values.push_back(id);
399}
400
408inline void appendMarker(std::vector<std::int32_t> &values, std::int32_t marker, std::size_t index)
409{
410 if (!marker && values.empty())
411 return;
412 if (values.empty())
413 values.resize(index, 0);
414 values.push_back(marker);
415}
416
422inline std::int64_t externalId(const std::vector<std::int64_t> &ids, std::size_t index)
423{
424 return ids.empty() ? std::int64_t(index + 1) : ids[index];
425}
426
431inline std::int64_t signed64(std::uint64_t bits)
432{
433 std::int64_t value;
434 std::memcpy(&value, &bits, 8);
435 return value;
436}
437
442inline std::int32_t signed32(std::uint64_t bits)
443{
444 std::uint32_t raw = std::uint32_t(bits);
445 std::int32_t value;
446 std::memcpy(&value, &raw, 4);
447 return value;
448}
449
454inline void setError(IOReport &r, const Stop &e)
455{
456 r.status = e.status;
457 r.code = e.code;
458 r.message = e.message;
459}
460
464inline void setException(IOReport &r, const std::bad_alloc &)
465{
467 r.code = "allocation";
468 r.message = "Insufficient memory.";
469}
470
475inline void setException(IOReport &r, const std::exception &e)
476{
478 r.code = "exception";
479 r.message = e.what();
480}
481
488inline void validated(const Mesh &mesh, const IOOptions &o)
489{
490 const auto r = validateMesh(mesh, o.limits, o.cancelled);
491 if (!r.complete())
492 fail(r.code, r.message, r.status);
493}
494}
std::uint64_t number(int width, bool hashed=true)
Read a little-endian unsigned value of width bytes.
Definition file.hpp:341
void raw(void *target, std::size_t n, bool hashed=true)
Read exactly n bytes.
Definition file.hpp:327
std::uint64_t hash
FNV-1a hash of the bytes read with hashed true, starting at hashSeed.
Definition file.hpp:284
std::uint64_t bytes
Bytes consumed through byte(), hashed or not.
Definition file.hpp:284
bool byte(unsigned char &value, bool hashed=true)
Read one byte, refilling the buffer when it is empty.
Definition file.hpp:301
double real()
Read 8 hashed bytes as a little-endian double bit pattern.
Definition file.hpp:355
std::string text(std::uint64_t &remaining)
Read a counted string (uint64 byte length, then bytes), charging a byte budget.
Definition file.hpp:369
Input(std::istream &stream, Work &work)
Bind the source and work budget.
Definition file.hpp:293
void raw(const void *data, std::size_t n, bool hashed=true)
Append bytes, flushing whenever the buffer fills.
Definition file.hpp:218
void flush()
Poll cancellation, then write out the buffered bytes.
Definition file.hpp:204
void text(const std::string &s)
Append a counted string: its uint64 byte length, then its bytes, both hashed.
Definition file.hpp:264
void number(std::uint64_t value, int width, bool hashed=true)
Append the low bytes of an unsigned value, least significant first.
Definition file.hpp:240
std::uint64_t bytes
Bytes accepted by raw(), hashed or not.
Definition file.hpp:191
std::uint64_t hash
FNV-1a hash of the bytes written with hashed true, starting at hashSeed.
Definition file.hpp:191
Output(std::FILE *file, Work &work)
Bind the destination and work budget.
Definition file.hpp:199
void real(double value)
Append a double's 8-byte bit pattern, little-endian and hashed.
Definition file.hpp:253
Transaction(const Transaction &)=delete
std::FILE * file() const
The open temporary, owned by this object (do not close it); null after commit().
Definition file.hpp:124
void commit()
Flush, optionally sync, close and publish the temporary at the target; call once.
Definition file.hpp:135
Transaction(const std::filesystem::path &target, const IOOptions &options)
Validate the target and create its exclusive temporary.
Definition file.hpp:73
~Transaction()
Close and remove the temporary unless commit() published it; errors ignored.
Definition file.hpp:113
The Qt-facing MeshKit API (meshkit::core_qt, meshkit::model_qt, meshkit::io_qt): mesh value types and...
Implementation helpers of the native jobs and codecs (failure signalling, work accounting,...
constexpr std::uint64_t hashSeed
64-bit FNV-1a offset basis, the starting value of every running hash.
Definition file.hpp:35
void appendMarker(std::vector< std::int32_t > &values, std::int32_t marker, std::size_t index)
Append one marker to a sparse marker array that stays empty while markers are 0.
Definition file.hpp:408
std::int64_t externalId(const std::vector< std::int64_t > &ids, std::size_t index)
External id of a record from a possibly empty sparse id array.
Definition file.hpp:422
void fail(std::string code, std::string message, RunStatus status=RunStatus::Infeasible)
Abort the current native operation by throwing a Stop.
Definition operation.hpp:32
bool validPath(const std::filesystem::path &p)
Reject empty paths and paths containing an embedded NUL character.
Definition file.hpp:29
std::int32_t signed32(std::uint64_t bits)
Reinterpret the low 32 bits of a pattern as a two's-complement signed value.
Definition file.hpp:442
void setException(IOReport &r, const std::bad_alloc &)
Record an allocation failure: status ResourceLimit, code allocation.
Definition file.hpp:464
void hashBytes(std::uint64_t &hash, const unsigned char *p, std::size_t n)
Fold bytes into a running 64-bit FNV-1a hash.
Definition file.hpp:42
void setError(IOReport &r, const Stop &e)
Copy a caught Stop into a report.
Definition file.hpp:454
void appendId(std::vector< std::int64_t > &values, std::int64_t id, std::size_t index)
Append one external id to a sparse id array that stays empty while ids are 1, 2, ....
Definition file.hpp:389
void validated(const Mesh &mesh, const IOOptions &o)
Run validateMesh() with the options' limits and cancellation poll.
Definition file.hpp:488
std::int64_t signed64(std::uint64_t bits)
Reinterpret a 64-bit pattern as a two's-complement signed value.
Definition file.hpp:431
OperationReport validateMesh(const Mesh &, const Limits &={}, const std::function< bool()> &cancelled={})
Check a mesh's local validity without modifying it.
Definition mesh.cpp:9
@ ResourceLimit
Stopped at a resource cap or allocation failure; no result.
Definition events.hpp:47
@ Failed
Stopped by an error such as an exception or I/O error; no result.
Definition events.hpp:49
Options shared by the MKB and 2DM readers and writers.
Definition io.hpp:27
Limits limits
Count, workspace-estimate, work, metadata-byte and 2DM line-length limits.
Definition io.hpp:29
std::function< bool()> cancelled
Optional poll; returning true aborts with status Cancelled (code cancelled).
Definition io.hpp:32
Outcome of one read or write: status, code, message, counts, timing and byte total.
Definition io.hpp:49
Neutral owned mesh: XYZ vertices, triangle/quad cells, marked edges, paths, optional external ids and...
Definition mesh.hpp:159
RunStatus status
Final status; Failed until the operation sets it.
Definition mesh.hpp:234
std::string message
Human-readable description of the outcome.
Definition mesh.hpp:242
std::string code
Short machine-readable outcome code, e.g.
Definition mesh.hpp:242
Exception object thrown by fail() to abort a native operation.
Definition operation.hpp:19
RunStatus status
Report status such as Failed, Infeasible, ResourceLimit or Cancelled.
Definition operation.hpp:20
std::string message
Human-readable English explanation.
Definition operation.hpp:22
std::string code
Short machine-readable failure code such as work_limit.
Definition operation.hpp:22
Work-unit budget, workspace-estimate gate and cancellation poll for one operation.
Definition operation.hpp:43