/* SPDX-FileCopyrightText: 2001-2002 NaN Holding BV. All rights reserved. * * SPDX-License-Identifier: GPL-2.0-or-later */ /** \file * \ingroup blenloader */ /** * FILE FORMAT * =========== * * IFF-style structure (but not IFF compatible!) * * Start of the file: * * Historic Blend-files (pre-Blender 5.0): * `BLENDER_V100` : Fixed 12 bytes length. See #BLEND_FILE_FORMAT_VERSION_0 for details. * * Current Blend-files (Blender 5.0 and later): * `BLENDER17-01v0500`: Variable bytes length. See #BLEND_FILE_FORMAT_VERSION_1 for details. * * data-blocks: (also see struct #BHead). *
 * `bh.code`       `char[4]` see `BLO_core_bhead.hh` for a list of known types.
 * `bh.len`        `int32` length data after #BHead in bytes.
 * `bh.old`        `void *` old pointer (the address at the time of writing the file).
 * `bh.SDNAnr`     `int32` struct index of structs stored in #DNA1 data.
 * `bh.nr`         `int32` in case of array: number of structs.
 * data
 * ...
 * ...
 * 
* * Almost all data in Blender are structures. Each struct saved * gets a BHead header. With BHead the struct can be linked again * and compared with #StructDNA. * * WRITE * ===== * * Preferred writing order: (not really a must, but why would you do it random?) * Any case: direct data is ALWAYS after the lib block. * * (Local file data) * - for each LibBlock * - write LibBlock * - write associated direct data * (External file data) * - per library * - write library block * - per LibBlock * - write the ID of LibBlock * - write #BLO_CODE_GLOB (#RenderInfo struct. 128x128 blend file preview is optional). * - write #BLO_CODE_GLOB (#FileGlobal struct) (some global vars). * - write #BLO_CODE_DNA1 (#SDNA struct) * - write #BLO_CODE_USER (#UserDef struct) for file paths: * - #BLENDER_STARTUP_FILE (on UNIX `~/.config/blender/X.X/config/startup.blend`). * - #BLENDER_USERPREF_FILE (on UNIX `~/.config/blender/X.X/config/userpref.blend`). */ #include #include #include #include #include #include #include #include #include #ifdef WIN32 # include "BLI_winstuff.h" # include "winsock2.h" # include #else # include /* FreeBSD, for write() and close(). */ #endif #include #include "BLI_utildefines.h" #include "CLG_log.h" /* Allow writefile to use deprecated functionality (for forward compatibility code). */ #define DNA_DEPRECATED_ALLOW #include "DNA_fileglobal_types.h" #include "DNA_genfile.h" #include "DNA_key_types.h" #include "DNA_print.hh" #include "DNA_sdna_pointers.hh" #include "DNA_sdna_types.h" #include "DNA_userdef_types.h" #include "DNA_windowmanager_types.h" #include "BLI_endian_defines.h" #include "BLI_fileops.hh" #include "BLI_implicit_sharing.hh" #include "BLI_listbase.h" #include "BLI_math_base.h" #include "BLI_math_matrix.h" #include "BLI_multi_value_map.hh" #include "BLI_path_utils.hh" #include "BLI_set.hh" #include "BLI_string.h" #include "BLI_threads.h" #include "BLI_time.h" #include "MEM_guardedalloc.h" #include "BKE_asset.hh" #include "BKE_blender_version.h" #include "BKE_bpath.hh" #include "BKE_global.hh" /* For #Global `G`. */ #include "BKE_idprop.hh" #include "BKE_idtype.hh" #include "BKE_layer.hh" #include "BKE_lib_id.hh" #include "BKE_lib_override.hh" #include "BKE_lib_query.hh" #include "BKE_library.hh" #include "BKE_main.hh" #include "BKE_main_namemap.hh" #include "BKE_node.hh" #include "BKE_packedFile.hh" #include "BKE_preferences.h" #include "BKE_report.hh" #include "BKE_workspace.hh" #include "DRW_engine.hh" #include "BLO_blend_validate.hh" #include "BLO_read_write.hh" #include "BLO_readfile.hh" #include "BLO_undofile.hh" #include "BLO_writefile.hh" #include "readfile.hh" #include "writefile.hh" #include namespace blender { /* Make preferences read-only. */ #define U (*((const UserDef *)&U)) /** * Generate an additional file next to every saved .blend file that contains the file content in a * more human readable form. */ #define GENERATE_DEBUG_BLEND_FILE 0 #define DEBUG_BLEND_FILE_SUFFIX ".debug.txt" /* ********* my write, buffered writing with minimum size chunks ************ */ /* Use optimal allocation since blocks of this size are kept in memory for undo. */ #define MEM_BUFFER_SIZE MEM_SIZE_OPTIMAL(1 << 17) /* 128kb */ #define MEM_CHUNK_SIZE MEM_SIZE_OPTIMAL(1 << 15) /* ~32kb */ #define ZSTD_BUFFER_SIZE (1 << 21) /* 2mb */ #define ZSTD_CHUNK_SIZE (1 << 20) /* 1mb */ #define ZSTD_COMPRESSION_LEVEL 3 static CLG_LogRef LOG = {"blend.writefile"}; static CLG_LogRef LOG_UNDO = {"undo"}; /** Use if we want to store how many bytes have been written to the file. */ // #define USE_WRITE_DATA_LEN /* -------------------------------------------------------------------- */ /** \name Internal Write Wrapper's (Abstracts Compression) * \{ */ struct ZstdFrame { ZstdFrame *next, *prev; uint32_t compressed_size; uint32_t uncompressed_size; }; class WriteWrap { public: virtual bool open(const char *filepath) = 0; virtual bool close() = 0; virtual bool write(const void *buf, size_t buf_len) = 0; /** Buffer output (we only want when output isn't already buffered). */ bool use_buf = true; }; class RawWriteWrap : public WriteWrap { public: bool open(const char *filepath) override; bool close() override; bool write(const void *buf, size_t buf_len) override; private: int file_handle = 0; }; class CallbackWriteWrap : public WriteWrap { public: CallbackWriteWrap(BLO_WriteFileCallback callback, void *user_data) : callback_(callback), user_data_(user_data) { } bool open(const char * /*filepath*/) override { return callback_ != nullptr; } bool close() override { return true; } bool write(const void *buf, const size_t buf_len) override { return callback_ != nullptr && callback_(buf, buf_len, user_data_); } private: BLO_WriteFileCallback callback_ = nullptr; void *user_data_ = nullptr; }; bool RawWriteWrap::open(const char *filepath) { int file; file = BLI_open(filepath, O_BINARY + O_WRONLY + O_CREAT + O_TRUNC, 0666); if (file != -1) { file_handle = file; return true; } return false; } bool RawWriteWrap::close() { return (::close(file_handle) != -1); } bool RawWriteWrap::write(const void *buf, size_t buf_len) { return ::write(file_handle, buf, buf_len) == buf_len; } struct ThreadSlot; class ZstdWriteWrap : public WriteWrap { struct ZstdWriteBlockTask; WriteWrap &base_wrap; ListBaseT threadpool = {}; ListBaseT tasks = {}; ThreadMutex mutex = {}; ThreadCondition condition = {}; int next_frame = 0; int num_frames = 0; ListBaseT frames = {}; bool write_error = false; public: ZstdWriteWrap(WriteWrap &base_wrap) : base_wrap(base_wrap) {} bool open(const char *filepath) override; bool close() override; bool write(const void *buf, size_t buf_len) override; private: void write_task(ZstdWriteBlockTask *task); void write_u32_le(uint32_t val); void write_seekable_frames(); }; struct ZstdWriteWrap::ZstdWriteBlockTask { ZstdWriteBlockTask *next, *prev; void *data; size_t size; int frame_number; ZstdWriteWrap *ww; static void *write_task(void *userdata) { auto *task = static_cast(userdata); task->ww->write_task(task); return nullptr; } }; void ZstdWriteWrap::write_task(ZstdWriteBlockTask *task) { size_t out_buf_len = ZSTD_compressBound(task->size); void *out_buf = MEM_new_uninitialized(out_buf_len, "Zstd out buffer"); size_t out_size = ZSTD_compress( out_buf, out_buf_len, task->data, task->size, ZSTD_COMPRESSION_LEVEL); MEM_delete_void(task->data); BLI_mutex_lock(&mutex); while (next_frame != task->frame_number) { BLI_condition_wait(&condition, &mutex); } if (ZSTD_isError(out_size)) { write_error = true; } else { if (base_wrap.write(out_buf, out_size)) { ZstdFrame *frameinfo = MEM_new_uninitialized("zstd frameinfo"); frameinfo->uncompressed_size = task->size; frameinfo->compressed_size = out_size; BLI_addtail(&frames, frameinfo); } else { write_error = true; } } next_frame++; BLI_mutex_unlock(&mutex); BLI_condition_notify_all(&condition); MEM_delete_void(out_buf); } bool ZstdWriteWrap::open(const char *filepath) { if (!base_wrap.open(filepath)) { return false; } /* Leave one thread open for the main writing logic, unless we only have one HW thread. */ int num_threads = max_ii(1, BLI_system_thread_count() - 1); BLI_threadpool_init(&threadpool, ZstdWriteBlockTask::write_task, num_threads); BLI_mutex_init(&mutex); BLI_condition_init(&condition); return true; } void ZstdWriteWrap::write_u32_le(uint32_t val) { /* NOTE: this is endianness-sensitive. * This value must always be written as little-endian. */ BLI_assert(ENDIAN_ORDER == L_ENDIAN); base_wrap.write(&val, sizeof(uint32_t)); } /* In order to implement efficient seeking when reading the .blend, we append * a skippable frame that encodes information about the other frames present * in the file. * The format here follows the upstream spec for seekable files: * https://github.com/facebook/zstd/blob/master/contrib/seekable_format/zstd_seekable_compression_format.md * If this information is not present in a file (e.g. if it was compressed * with external tools), it can still be opened in Blender, but seeking will * not be supported, so more memory might be needed. */ void ZstdWriteWrap::write_seekable_frames() { /* Write seek table header (magic number and frame size). */ write_u32_le(0x184D2A5E); /* The actual frame number might not match num_frames if there was a write error. */ const uint32_t num_frames = frames.count(); /* Each frame consists of two u32, so 8 bytes each. * After the frames, a footer containing two u32 and one byte (9 bytes total) is written. */ const uint32_t frame_size = num_frames * 8 + 9; write_u32_le(frame_size); /* Write seek table entries. */ for (ZstdFrame &frame : frames) { write_u32_le(frame.compressed_size); write_u32_le(frame.uncompressed_size); } /* Write seek table footer (number of frames, option flags and second magic number). */ write_u32_le(num_frames); const char flags = 0; /* We don't store checksums for each frame. */ base_wrap.write(&flags, 1); write_u32_le(0x8F92EAB1); } bool ZstdWriteWrap::close() { BLI_threadpool_end(&threadpool); tasks.free_no_destruct(); BLI_mutex_end(&mutex); BLI_condition_end(&condition); write_seekable_frames(); frames.free_no_destruct(); return base_wrap.close() && !write_error; } bool ZstdWriteWrap::write(const void *buf, const size_t buf_len) { if (write_error) { return false; } ZstdWriteBlockTask *task = MEM_new_uninitialized(__func__); task->data = MEM_new_uninitialized(buf_len, __func__); memcpy(task->data, buf, buf_len); task->size = buf_len; task->frame_number = num_frames++; task->ww = this; BLI_mutex_lock(&mutex); BLI_addtail(&tasks, task); /* If there's a free worker thread, just push the block into that thread. * Otherwise, we wait for the earliest thread to finish. * We look up the earliest thread while holding the mutex, but release it * before joining the thread to prevent a deadlock. */ ZstdWriteBlockTask *first_task = static_cast(tasks.first); BLI_mutex_unlock(&mutex); if (!BLI_available_threads(&threadpool)) { BLI_threadpool_remove(&threadpool, first_task); /* If the task list was empty before we pushed our task, there should * always be a free thread. */ BLI_assert(first_task != task); BLI_remlink(&tasks, first_task); MEM_delete(first_task); } BLI_threadpool_insert(&threadpool, task); return true; } /** \} */ /* -------------------------------------------------------------------- */ /** \name Write Data Type & Functions * \{ */ static WriteData *writedata_new(WriteWrap *ww) { WriteData *wd = MEM_new(__func__); wd->timestamp_init = BLI_time_now_seconds(); wd->sdna = DNA_sdna_current_get(); wd->stable_address_ids.sdna_pointers = std::make_unique(*wd->sdna); wd->ww = ww; if ((ww == nullptr) || (ww->use_buf)) { if (ww == nullptr) { wd->buffer.max_size = MEM_BUFFER_SIZE; wd->buffer.chunk_size = MEM_CHUNK_SIZE; } else { wd->buffer.max_size = ZSTD_BUFFER_SIZE; wd->buffer.chunk_size = ZSTD_CHUNK_SIZE; } wd->buffer.buf = MEM_new_array_uninitialized(wd->buffer.max_size, "wd->buffer.buf"); } return wd; } static void writedata_do_write(WriteData *wd, const void *mem, const size_t memlen) { if ((wd == nullptr) || wd->validation_data.critical_error || (mem == nullptr) || memlen < 1) { return; } if (memlen > INT_MAX) { BLI_assert_msg(0, "Cannot write chunks bigger than INT_MAX."); return; } if (UNLIKELY(wd->validation_data.critical_error)) { return; } /* Memory based save. */ if (wd->use_memfile) { BLO_memfile_chunk_add(&wd->mem, static_cast(mem), memlen); } else { if (!wd->ww->write(mem, memlen)) { wd->validation_data.critical_error = true; } } } static void writedata_free(WriteData *wd) { if (wd->buffer.buf) { MEM_delete(wd->buffer.buf); } MEM_delete(wd); } /** \} */ /* -------------------------------------------------------------------- */ /** \name Local Writing API 'mywrite' * \{ */ /** * Flush helps the de-duplicating memory for undo-save by logically segmenting data, * so differences in one part of memory won't cause unrelated data to be duplicated. */ static void mywrite_flush(WriteData *wd) { if (wd->buffer.used_len != 0) { writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len); wd->buffer.used_len = 0; } } /** * Low level WRITE(2) wrapper that buffers data * \param adr: Pointer to new chunk of data * \param len: Length of new chunk of data */ static void mywrite(WriteData *wd, const void *adr, size_t len) { if (UNLIKELY(wd->validation_data.critical_error)) { return; } if (UNLIKELY(adr == nullptr)) { BLI_assert(0); return; } #ifdef USE_WRITE_DATA_LEN wd->write_len += len; #endif if (wd->buffer.buf == nullptr) { writedata_do_write(wd, adr, len); } else { /* If we have a single big chunk, write existing data in * buffer and write out big chunk in smaller pieces. */ if (len > wd->buffer.chunk_size) { if (wd->buffer.used_len != 0) { writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len); wd->buffer.used_len = 0; } do { const size_t writelen = std::min(len, wd->buffer.chunk_size); writedata_do_write(wd, adr, writelen); adr = static_cast(adr) + writelen; len -= writelen; } while (len > 0); return; } /* If data would overflow buffer, write out the buffer. */ if (len + wd->buffer.used_len > wd->buffer.max_size - 1) { writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len); wd->buffer.used_len = 0; } /* Append data at end of buffer. */ memcpy(&wd->buffer.buf[wd->buffer.used_len], adr, len); wd->buffer.used_len += len; } } /** * BeGiN initializer for mywrite * \param ww: File write wrapper. * \param compare: Previous memory file (can be nullptr). * \param current: The current memory file (can be nullptr). * \warning Talks to other functions with global parameters */ static WriteData *mywrite_begin(WriteWrap *ww, MemFile *compare, MemFile *current) { WriteData *wd = writedata_new(ww); if (current != nullptr) { BLO_memfile_write_init(wd, &wd->mem, current, compare); } return wd; } /** * END the mywrite wrapper * \return True if write failed * \return unknown global variable otherwise * \warning Talks to other functions with global parameters */ static bool mywrite_end(WriteData *wd) { if (wd->buffer.used_len != 0) { writedata_do_write(wd, wd->buffer.buf, wd->buffer.used_len); wd->buffer.used_len = 0; } if (wd->use_memfile) { BLO_memfile_write_finalize(wd, &wd->mem); CLOG_INFO(&LOG_UNDO, "Memfile undo step written in %.3f seconds", BLI_time_now_seconds() - wd->timestamp_init); } else { CLOG_INFO( &LOG, "Blendfile written in %.3f seconds", BLI_time_now_seconds() - wd->timestamp_init); } const bool err = wd->validation_data.critical_error; writedata_free(wd); return err; } static uint64_t get_stable_pointer_hint_for_id(const ID &id, const bool is_undo) { /* Make the stable pointer depend on a specific data of the ID. * Note that this is different when writing blendfile on disk, and when writing an undo step in * memory (memfile). * * For the blendfile on disk, the ID name is used, together with its library if linked, as this * is effectively the 'unique identifier' of IDs in blend-files and across linking, * so if these change, it's also fine to get a different 'stable pointer'. * * For the undo memfile however, things are different: It is possible that a same ID name is * reused for two different IDs in two different consecutive undo steps (see #149899). Getting * the same stable pointer in this case can lead to falsely detecting other IDs using these as * unchanged, leading to undo data corruption and crashes. * In this case, using the session UID is a better source of info, as these are assumed unique * during an editing session, and are extremely stable for a same ID (even if it is e.g. * renamed). */ if (is_undo) { /* Note: Using the uint32_t session_uid also means that library data can be ignored (and ID * made local always get a new session UID), and that there is no need to call the hashing code * at all. * * However, to leave enough 'address space' for all the sub-data pointers, its value is shifted * into higher significant bits of the returned value (only shift by 20 bits here, since * #stable_id_from_hint also shifts further the generated values by 4, and some of the most * significant bits are also reserved for flags, like the #generated_address_id_on_undo_flag * one). */ return uint64_t(id.session_uid) << 20; } const uint64_t id_hash = XXH3_64bits(id.name, strlen(id.name)); if (!id.lib) { return id_hash; } const uint64_t lib_hash = XXH3_64bits(id.lib->id.name, strlen(id.lib->id.name)); return id_hash ^ lib_hash; } /** * Start writing of data related to a single ID. * * Only does something when storing an undo step. */ static void mywrite_id_begin(WriteData *wd, ID *id) { BLI_assert(wd->is_writing_id == false); wd->is_writing_id = true; BLI_assert(wd->validation_data.per_id_addresses_set.is_empty()); BLI_assert_msg(ID_IS_PACKED(id) || id->deep_hash.is_null(), "The only IDs with non-null deep-hash data should be packed linked ones"); BLI_assert_msg((id->flag & ID_FLAG_EMBEDDED_DATA) == 0 || id->deep_hash.is_null(), "Embedded IDs should always have a null deep-hash data"); const bool is_undo = wd->use_memfile; wd->stable_address_ids.next_id_hint = get_stable_pointer_hint_for_id(*id, is_undo); if (is_undo) { wd->mem.current_id_session_uid = id->session_uid; /* If current next memchunk does not match the ID we are about to write, or is not the _first_ * one for said ID, try to find the correct memchunk in the mapping using ID's session_uid. */ const MemFileChunk *curr_memchunk = wd->mem.reference_current_chunk; const MemFileChunk *prev_memchunk = curr_memchunk != nullptr ? static_cast(curr_memchunk->prev) : nullptr; if (curr_memchunk == nullptr || curr_memchunk->id_session_uid != id->session_uid || (prev_memchunk != nullptr && (prev_memchunk->id_session_uid == curr_memchunk->id_session_uid))) { if (MemFileChunk *ref = wd->mem.id_session_uid_mapping.lookup_default(id->session_uid, nullptr)) { wd->mem.reference_current_chunk = ref; } /* Else, no existing memchunk found, i.e. this is supposed to be a new ID. */ } /* Otherwise, we try with the current memchunk in any case, whether it is matching current * ID's session_uid or not. */ } } /** * Start writing of data related to a single ID. * * Only does something when storing an undo step. */ static void mywrite_id_end(WriteData *wd, ID * /*id*/) { if (wd->use_memfile) { /* Very important to do it after every ID write now, otherwise we cannot know whether a * specific ID changed or not. */ mywrite_flush(wd); wd->mem.current_id_session_uid = MAIN_ID_SESSION_UID_UNSET; /* In undo case: * - IDs never get a stable address id; * - Shared data (implicit sharing) never gets a stable address id. * - Other addresses (private ID data) should never be shared across IDs. * * So we can clear the stable address ids after each ID writing. This makes the mapping even * smaller, and ensures that data dynamically generated on write (which may re-use the same * addresses between different IDs) do not trigger the assert in * `BLO_write_generated_pointer_tag`. * * Note that `WriteDataStableAddressIDs::used_ids` and * `WriteDataStableAddressIDs::next_id_hint` are kept, to ensure generated address ids are * never re-used in another ID (not strictly necessary, but can helps with debugging etc.). */ wd->stable_address_ids.pointer_map.clear(); } wd->validation_data.per_id_addresses_set.clear(); wd->per_id_written_shared_addresses.clear(); BLI_assert(wd->is_writing_id == true); wd->is_writing_id = false; } /** \} */ /* -------------------------------------------------------------------- */ /** \name Generic DNA File Writing * \{ */ /** * Return \a false if the given 'old' address is not valid in current context. The block should * not be written in that case. * * \note Currently only checks that #BLO_CODE_DATA blocks written as part of an ID data never match * an already written one for the same ID. */ static bool write_at_address_validate(WriteData *wd, const int filecode, const void *address) { /* Skip in undo case. */ if (wd->use_memfile) { return true; } if (wd->is_writing_id && filecode == BLO_CODE_DATA) { if (!wd->validation_data.per_id_addresses_set.add(address)) { CLOG_ERROR(&LOG, "Same identifier (old address) used several times for a same ID, skipping this " "block to avoid critical corruption of the Blender file."); return false; } } return true; } static void write_bhead(WriteData *wd, const BHead &bhead) { if constexpr (sizeof(void *) == 4) { /* Always write #BHead4 in 32 bit builds. */ BHead4 bh; bh.code = bhead.code; bh.old = uint32_t(uintptr_t(bhead.old)); bh.nr = bhead.nr; bh.SDNAnr = bhead.SDNAnr; bh.len = bhead.len; mywrite(wd, &bh, sizeof(bh)); return; } /* Write new #LargeBHead8 headers if enabled. Older Blender versions can't read those. */ if (!USER_DEVELOPER_TOOL_TEST(&U, write_legacy_blend_file_format)) { if (SYSTEM_SUPPORTS_WRITING_FILE_VERSION_1) { static_assert(sizeof(BHead) == sizeof(LargeBHead8)); mywrite(wd, &bhead, sizeof(bhead)); return; } } /* Write older #SmallBHead8 headers so that Blender versions that don't support #LargeBHead8 can * read the file. */ SmallBHead8 bh; bh.code = bhead.code; bh.old = uint64_t(bhead.old); bh.nr = bhead.nr; bh.SDNAnr = bhead.SDNAnr; bh.len = bhead.len; /* Check that the written buffer size is compatible with the limits of #SmallBHead8. */ if (bhead.len > std::numeric_limits::max()) { CLOG_ERROR(&LOG, "Written .blend file is corrupt, because a memory block is too large."); return; } mywrite(wd, &bh, sizeof(bh)); } /** * This bit is used to mark address ids that are generated for pointers during undo (when writing * undo steps, most addresses are used as-is). */ constexpr uint64_t generated_address_id_on_undo_flag = uint64_t(1) << 63; /** Mask to remove bits form the generated hash values used as stable addresses. */ constexpr uint64_t generated_address_id_mask = generated_address_id_on_undo_flag; static uint64_t stable_id_from_hint(const uint64_t hint) { /* Add a stride. This is not strictly necessary but may help with debugging later on because it's * easier to identify bad ids. */ uint64_t stable_id = hint << 4; if (stable_id == 0) { /* Null values are reserved for nullptr. */ stable_id = (1 << 4); } /* Remove the first bits, which are reserved for certain types of generated values (like * actual stable ids used in some cases when writing undo steps). */ stable_id &= ~generated_address_id_mask; return stable_id; } static uint64_t get_next_stable_address_id(WriteData &wd, uint64_t &hint) { uint64_t stable_id = stable_id_from_hint(hint); while (!wd.stable_address_ids.used_ids.add(stable_id)) { /* Generate a new hint because there is a collision. Collisions are generally expected to be * very rare. It can happen when #get_stable_pointer_hint_for_id produces values that are very * close for different IDs. */ hint = XXH3_64bits(&hint, sizeof(uint64_t)); stable_id = stable_id_from_hint(hint); } hint++; return stable_id; } static uint64_t get_address_id_int(WriteData &wd, const void *address) { if (address == nullptr) { return 0; } /* In undo case, addresses are kept as-is, unless they have been tagged by specific functions * like `BLO_write_generated_pointer_tag`, in which case their value will already be in the * `pointer_map`. */ if (wd.use_memfile) { return wd.stable_address_ids.pointer_map.lookup_default_as( address, reinterpret_cast(address)); } /* Either reuse an existing identifier or create a new one. */ return wd.stable_address_ids.pointer_map.lookup_or_add_cb(address, [&]() { return get_next_stable_address_id(wd, wd.stable_address_ids.next_id_hint); }); } static const void *get_address_id(WriteData &wd, const void *address) { return reinterpret_cast(get_address_id_int(wd, address)); } /** * When writing an undo step, most pointers do not use generated stable addresses, since by * definition if a pointer address changes, it owning data is also changed, and getting a stable * value here might actually lead to _preventing_ proper change detection. * * However, there are a few pointers, used for data generated at runtime as part of the writefile * process, that do need to be kept 'stable', since by definition they change on every write * operation. */ static uint64_t get_address_id_for_undo(WriteData &wd) { return get_next_stable_address_id(wd, wd.stable_address_ids.next_id_hint) | generated_address_id_on_undo_flag; } static void writestruct_at_address_nr(WriteData *wd, const int filecode, const int struct_nr, const int64_t nr, const void *adr, const void *data) { BLI_assert(struct_nr > 0 && struct_nr <= dna::sdna_struct_id_get_max()); if (adr == nullptr || data == nullptr || nr == 0) { return; } if (!write_at_address_validate(wd, filecode, adr)) { return; } const int64_t len_in_bytes = nr * DNA_struct_size(wd->sdna, struct_nr); if (!SYSTEM_SUPPORTS_WRITING_FILE_VERSION_1 || USER_DEVELOPER_TOOL_TEST(&U, write_legacy_blend_file_format)) { if (len_in_bytes > INT32_MAX) { CLOG_ERROR(&LOG, "Cannot write chunks bigger than INT_MAX."); return; } } /* Get the address identifier that will be written to the file. */ const void *address_id = get_address_id(*wd, adr); const void *data_to_write; DynamicStackBuffer<16 * 1024> buffer_owner(len_in_bytes, 64); const dna::pointers::StructInfo &struct_info = wd->stable_address_ids.sdna_pointers->get_for_struct(struct_nr); const bool can_write_raw_runtime_data = struct_info.pointers.is_empty(); if (can_write_raw_runtime_data) { /* The passed in data contains no pointers, so it can be written without an additional copy. */ data_to_write = data; } else { void *buffer = buffer_owner.buffer(); data_to_write = buffer; memcpy(buffer, data, len_in_bytes); /* Overwrite pointers with their corresponding address identifiers. */ for (const int i : IndexRange(nr)) { for (const dna::pointers::PointerInfo &pointer_info : struct_info.pointers) { const int offset = i * struct_info.size_in_bytes + pointer_info.offset; const void **p_ptr = reinterpret_cast(POINTER_OFFSET(buffer, offset)); const void *p_ptr_address_id = get_address_id(*wd, *p_ptr); *p_ptr = p_ptr_address_id; } } } BHead bh; bh.code = filecode; bh.old = address_id; bh.nr = nr; bh.SDNAnr = struct_nr; bh.len = len_in_bytes; if (bh.len == 0) { return; } if (wd->debug_dst) { dna::print_structs_at_address( *wd->sdna, struct_nr, data_to_write, address_id, nr, *wd->debug_dst); } write_bhead(wd, bh); mywrite(wd, data_to_write, size_t(bh.len)); } static void writestruct_nr( WriteData *wd, const int filecode, const int struct_nr, const int64_t nr, const void *adr) { writestruct_at_address_nr(wd, filecode, struct_nr, nr, adr, adr); } static void write_raw_data_in_debug_file(WriteData *wd, const size_t len, const void *address_id, const void *data) { fmt::memory_buffer buf; fmt::appender dst{buf}; fmt::format_to(dst, " at {} ({} bytes)\n", address_id, len); constexpr int bytes_per_row = 8; const int len_digits = std::to_string(std::max(0, len - 1)).size(); for (size_t i = 0; i < len; i++) { if (i % bytes_per_row == 0) { fmt::format_to(dst, " {:{}}: ", i, len_digits); } fmt::format_to(dst, "{:02x} ", reinterpret_cast(data)[i]); if (i % bytes_per_row == bytes_per_row - 1) { fmt::format_to(dst, "\n"); } } if (len % bytes_per_row != 0) { fmt::format_to(dst, "\n"); } *wd->debug_dst << fmt::to_string(buf); } /** * \warning Do not use for structs. */ static void writedata( WriteData *wd, const int filecode, const void *data, const size_t len, const void *adr) { if (data == nullptr || len == 0) { return; } if (!write_at_address_validate(wd, filecode, adr)) { return; } if ((!SYSTEM_SUPPORTS_WRITING_FILE_VERSION_1 || USER_DEVELOPER_TOOL_TEST(&U, write_legacy_blend_file_format)) && len > INT_MAX) { BLI_assert_msg(0, "Cannot write chunks bigger than INT_MAX."); return; } const void *address_id = get_address_id(*wd, adr); BHead bh; bh.code = filecode; bh.old = address_id; bh.nr = 1; BLI_STATIC_ASSERT(SDNA_RAW_DATA_STRUCT_INDEX == 0, "'raw data' SDNA struct index should be 0") bh.SDNAnr = SDNA_RAW_DATA_STRUCT_INDEX; bh.len = int64_t(len); if (wd->debug_dst) { write_raw_data_in_debug_file(wd, len, address_id, adr); } write_bhead(wd, bh); mywrite(wd, data, len); } static void writedata(WriteData *wd, const int filecode, const size_t len, const void *adr) { writedata(wd, filecode, adr, len, adr); } /** * Use this to force writing of lists in same order as reading (using link_list). */ static void writelist_nr(WriteData *wd, const int filecode, const int struct_nr, const ListBase *lb) { const Link *link = static_cast(lb->first); while (link) { writestruct_nr(wd, filecode, struct_nr, 1, link); link = link->next; } } #if 0 static void writelist_id(WriteData *wd, const int filecode, const char *structname, const ListBase *lb) { const Link *link = lb->first; if (link) { const int struct_nr = DNA_struct_find_with_alias(wd->sdna, structname); if (struct_nr == -1) { printf("error: cannot find SDNA code <%s>\n", structname); return; } while (link) { writestruct_nr(wd, filecode, struct_nr, 1, link); link = link->next; } } } #endif #define writestruct_at_address(wd, filecode, struct_id, nr, adr, data) \ writestruct_at_address_nr(wd, filecode, dna::sdna_struct_id_get(), nr, adr, data) #define writestruct(wd, filecode, struct_id, nr, adr) \ writestruct_nr(wd, filecode, dna::sdna_struct_id_get(), nr, adr) /** \} */ /* -------------------------------------------------------------------- */ /** \name Typed DNA File Writing * * These functions are used by blender's .blend system for file saving/loading. * \{ */ /** * Take care using 'use_active_win', since we won't want the currently active window * to change which scene renders (currently only used for undo). */ static void current_screen_compat(Main *mainvar, const bool use_active_win, bScreen **r_screen, Scene **r_scene, ViewLayer **r_view_layer) { wmWindowManager *wm; wmWindow *window = nullptr; /* Find a global current screen in the first open window, to have * a reasonable default for reading in older versions. */ wm = static_cast(mainvar->wm.first); if (wm) { if (use_active_win) { /* Write the active window into the file, needed for multi-window undo #43424. */ for (window = static_cast(wm->windows.first); window; window = window->next) { if (window->active) { break; } } /* Fallback. */ if (window == nullptr) { window = static_cast(wm->windows.first); } } else { window = static_cast(wm->windows.first); } } *r_screen = (window) ? BKE_workspace_active_screen_get(window->workspace_hook) : nullptr; *r_scene = (window) ? window->scene : nullptr; *r_view_layer = (window && *r_scene) ? BKE_view_layer_find(*r_scene, window->view_layer_name) : nullptr; } struct RenderInfo { int sfra; int efra; char scene_name[MAX_ID_NAME - 2]; }; /** * This was originally added for the historic render-daemon feature, * now write because it can be easily extracted without reading the whole blend file. * * See: `scripts/modules/blend_render_info.py` */ static void write_renderinfo(WriteData *wd, Main *mainvar) { bScreen *curscreen; Scene *curscene = nullptr; ViewLayer *view_layer; /* XXX: in future, handle multiple windows with multiple screens? */ current_screen_compat(mainvar, false, &curscreen, &curscene, &view_layer); for (Scene &sce : mainvar->scenes) { if (!ID_IS_LINKED(&sce) && (&sce == curscene || (sce.r.scemode & R_BG_RENDER))) { RenderInfo data; data.sfra = sce.r.sfra; data.efra = sce.r.efra; memset(data.scene_name, 0, sizeof(data.scene_name)); STRNCPY(data.scene_name, sce.id.name + 2); writedata(wd, BLO_CODE_REND, sizeof(data), &data); } } } static void write_keymapitem(BlendWriter *writer, const wmKeyMapItem *kmi) { writer->write_struct(kmi); if (kmi->properties) { IDP_BlendWrite(writer, kmi->properties); } } static void write_userdef(BlendWriter *writer, const UserDef *userdef) { writestruct(writer->wd, BLO_CODE_USER, UserDef, 1, userdef); for (const bTheme &btheme : userdef->themes) { writer->write_struct(&btheme); } for (const wmKeyMap &keymap : userdef->user_keymaps) { writer->write_struct(&keymap); for (const wmKeyMapDiffItem &kmdi : keymap.diff_items) { writer->write_struct(&kmdi); if (kmdi.remove_item) { write_keymapitem(writer, kmdi.remove_item); } if (kmdi.add_item) { write_keymapitem(writer, kmdi.add_item); } } for (const wmKeyMapItem &kmi : keymap.items) { write_keymapitem(writer, &kmi); } } for (const wmKeyConfigPref &kpt : userdef->user_keyconfig_prefs) { writer->write_struct(&kpt); if (kpt.prop) { IDP_BlendWrite(writer, kpt.prop); } } for (const bUserMenu &um : userdef->user_menus) { writer->write_struct(&um); for (const bUserMenuItem &umi : um.items) { if (umi.type == USER_MENU_TYPE_OPERATOR) { const bUserMenuItem_Op *umi_op = reinterpret_cast(&umi); writer->write_struct(umi_op); if (umi_op->prop) { IDP_BlendWrite(writer, umi_op->prop); } } else if (umi.type == USER_MENU_TYPE_MENU) { const bUserMenuItem_Menu *umi_mt = reinterpret_cast(&umi); writer->write_struct(umi_mt); } else if (umi.type == USER_MENU_TYPE_PROP) { const bUserMenuItem_Prop *umi_pr = reinterpret_cast(&umi); writer->write_struct(umi_pr); } else { writer->write_struct(&umi); } } } for (const bAddon &bext : userdef->addons) { writer->write_struct(&bext); if (bext.prop) { IDP_BlendWrite(writer, bext.prop); } } for (const bPathCompare &path_cmp : userdef->autoexec_paths) { writer->write_struct(&path_cmp); } for (const bUserScriptDirectory &script_dir : userdef->script_directories) { writer->write_struct(&script_dir); } for (const bUserAssetLibrary &asset_library_ref : userdef->asset_libraries) { writer->write_struct(&asset_library_ref); } for (const bUserExtensionRepo &repo_ref : userdef->extension_repos) { writer->write_struct(&repo_ref); BKE_preferences_extension_repo_write_data(writer, &repo_ref); } for (const bUserAssetShelfSettings &shelf_settings : userdef->asset_shelves_settings) { writer->write_struct(&shelf_settings); BKE_asset_catalog_path_list_blend_write(writer, shelf_settings.enabled_catalog_paths); } for (const uiStyle &style : userdef->uistyles) { writer->write_struct(&style); } } /** * Writes ID and all its direct data to the file. */ static void write_id(WriteData *wd, ID *id) { const IDTypeInfo *id_type = BKE_idtype_get_info_from_id(id); mywrite_id_begin(wd, id); if (id_type->blend_write != nullptr) { BlendWriter writer = {wd}; BLO_Write_IDBuffer id_buffer{*id, wd->use_memfile, false}; id_type->blend_write(&writer, id_buffer.get(), id); } mywrite_id_end(wd, id); } static void write_id_placeholder(WriteData *wd, ID *id) { mywrite_id_begin(wd, id); /* Only copy required data for the placeholder ID. */ BLO_Write_IDBuffer id_buffer{*id, wd->use_memfile, true}; writestruct_at_address(wd, ID_LINK_PLACEHOLDER, ID, 1, id, &id_buffer); mywrite_id_end(wd, id); } /** Keep it last of `write_*_data` functions. */ static void write_libraries(WriteData *wd, Main *bmain) { const bool is_undo = wd->use_memfile; /* Gather IDs coming from each library. */ MultiValueMap linked_ids_by_library; { ID *id; FOREACH_MAIN_ID_BEGIN (bmain, id) { if (!ID_IS_LINKED(id)) { continue; } BLI_assert(id->lib); linked_ids_by_library.add(id->lib, id); } FOREACH_MAIN_ID_END; } Set written_libraries; for (Library &library_ptr : bmain->libraries) { Library &library = library_ptr; const Span ids = linked_ids_by_library.lookup(&library); /* Gather IDs that are somehow directly referenced by data in the current blend file. */ Vector ids_used_from_library; if (is_undo) { /* Always write placeholders for all linked IDs in undo case. This allows to properly remove * linked data that should not exist on undo/redo. See also #read_undo_move_libmain_data, * #read_libblock_undo_restore_linked and #read_undo_libraries_cleanup_unused_ids. */ ids_used_from_library = ids; } else { for (ID *id : ids) { if (id->us == 0) { continue; } if (ID_IS_PACKED(id)) { BLI_assert(library.flag & LIBRARY_FLAG_IS_ARCHIVE); ids_used_from_library.append(id); continue; } if (id->tag & ID_TAG_EXTERN) { ids_used_from_library.append(id); continue; } if ((id->tag & ID_TAG_INDIRECT) && (id->flag & ID_FLAG_INDIRECT_WEAK_LINK)) { ids_used_from_library.append(id); continue; } } } bool should_write_library = false; if (library.packedfile) { should_write_library = true; } else if (!library.runtime->archived_libraries.is_empty()) { /* Reference 'real' blendfile library of archived 'copies' of it containing packed linked * IDs should always be written. */ /* FIXME: A bit weak, as it could be that all archive libs are now empty (if all related * packed linked IDs have been deleted e.g.)... * Could be fixed by either adding more checks here, or ensuring empty archive libs are * deleted when no ID uses them anymore? */ should_write_library = true; } else if (is_undo) { /* When writing undo step we always write all existing libraries. That makes reading undo * step much easier when dealing with purely indirectly used libraries. */ should_write_library = true; } else { should_write_library = !ids_used_from_library.is_empty(); } if (!should_write_library) { /* Nothing from the library is used, so it does not have to be written. */ continue; } /* Since code below checking that archive libraries' parents have already been written, may * forcefully write that parent library in some cases, also double-check here that current * library has not yet been written. * * Note: In theory this should never be the case, as a parent library is always expected to be * written before its archives. */ if (written_libraries.contains(&library)) { CLOG_ERROR( &LOG, "Attempt to re-write already written library '%s', skipping", library.id.name); continue; } if (library.flag & LIBRARY_FLAG_IS_ARCHIVE) { if (!library.archive_parent_library) { CLOG_ERROR(&LOG, "Written archive library '%s' has no parent library", library.id.name); } if (!written_libraries.contains(library.archive_parent_library)) { CLOG_ERROR( &LOG, "Written archive library '%s', while its parent library '%s' has not been written", library.id.name, library.archive_parent_library->id.name); /* Only write the parent library itself, if it was not written so far, none of its IDs was * to be written either. */ write_id(wd, &library.archive_parent_library->id); written_libraries.add(library.archive_parent_library); } } write_id(wd, &library.id); written_libraries.add(&library); /* Write placeholders for linked data-blocks that are used, and real IDs for the packed linked * ones. */ for (ID *id : ids_used_from_library) { if (ID_IS_PACKED(id)) { write_id(wd, id); } else { /* In undo case, all existing linked IDs get a placeholder, even the ones not directly * linkable. */ if (!is_undo && !BKE_idtype_idcode_is_linkable(GS(id->name))) { CLOG_ERROR(&LOG, "Data-block '%s' from lib '%s' is not linkable, but is flagged as " "directly linked", id->name, library.runtime->filepath_abs); } write_id_placeholder(wd, id); } } } mywrite_flush(wd); } #ifdef WITH_BUILDINFO extern "C" ulong build_commit_timestamp; extern "C" char build_hash[]; #endif /** * Context is usually defined by WM, two cases where no WM is available: * - for forward compatibility, `curscreen` has to be saved * - for undo-file, `curscene` needs to be saved. */ static void write_global(WriteData *wd, const int fileflags, Main *mainvar) { const bool is_undo = wd->use_memfile; FileGlobal fg; bScreen *screen; Scene *scene; ViewLayer *view_layer; char subvstr[8]; /* Prevent memory checkers from complaining. */ memset(fg._pad, 0, sizeof(fg._pad)); memset(fg.filepath, 0, sizeof(fg.filepath)); memset(fg.build_hash, 0, sizeof(fg.build_hash)); fg._pad1 = nullptr; current_screen_compat(mainvar, is_undo, &screen, &scene, &view_layer); /* XXX: still remap `G`. */ fg.curscreen = screen; fg.curscene = scene; fg.cur_view_layer = view_layer; STRNCPY(fg.colorspace_scene_linear_name, mainvar->colorspace.scene_linear_name); copy_m3_m3(fg.colorspace_scene_linear_to_xyz, mainvar->colorspace.scene_linear_to_xyz.ptr()); /* Prevent to save this, is not good convention, and feature with concerns. */ fg.fileflags = (fileflags & ~G_FILE_FLAG_ALL_RUNTIME); fg.globalf = G.f; /* Write information needed for recovery. */ if (fileflags & G_FILE_RECOVER_WRITE) { STRNCPY(fg.filepath, mainvar->filepath); /* Compression is often turned of when writing recovery files. However, when opening the file, * it should be enabled again. */ fg.fileflags = G.fileflags & G_FILE_COMPRESS; } SNPRINTF(subvstr, "%4d", BLENDER_FILE_SUBVERSION); memcpy(fg.subvstr, subvstr, 4); fg.subversion = BLENDER_FILE_SUBVERSION; fg.minversion = BLENDER_FILE_MIN_VERSION; fg.minsubversion = BLENDER_FILE_MIN_SUBVERSION; #ifdef WITH_BUILDINFO /* TODO(sergey): Add branch name to file as well? */ fg.build_commit_timestamp = build_commit_timestamp; STRNCPY(fg.build_hash, build_hash); #else fg.build_commit_timestamp = 0; STRNCPY(fg.build_hash, "unknown"); #endif writestruct(wd, BLO_CODE_GLOB, FileGlobal, 1, &fg); } /** * Preview image, first 2 values are width and height * second are an RGBA image (uchar). * \note this uses 'TEST' since new types will segfault on file load for older blender versions. */ static void write_thumb(WriteData *wd, const BlendThumbnail *thumb) { if (thumb) { writedata(wd, BLO_CODE_TEST, BLEN_THUMB_MEMSIZE_FILE(thumb->width, thumb->height), thumb); } } /** \} */ /* -------------------------------------------------------------------- */ /** \name File Writing (Private) * \{ */ BLO_Write_IDBuffer::BLO_Write_IDBuffer(ID &id, const bool is_undo, const bool is_placeholder) : buffer_(is_placeholder ? sizeof(ID) : BKE_idtype_get_info_from_id(&id)->struct_size, alignof(ID)) { const IDTypeInfo *id_type = BKE_idtype_get_info_from_id(&id); ID *temp_id = static_cast(buffer_.buffer()); if (is_undo) { /* Record the changes that happened up to this undo push in * recalc_up_to_undo_push, and clear `recalc_after_undo_push` again * to start accumulating for the next undo push. */ id.recalc_up_to_undo_push = id.recalc_after_undo_push; id.recalc_after_undo_push = 0; } /* Copy ID data itself into buffer, to be able to freely modify it. */ if (is_placeholder) { /* For placeholders (references to linked data), zero-initialize, and only explicitly copy the * very small subset of required data. */ *temp_id = ID{}; temp_id->lib = id.lib; STRNCPY(temp_id->name, id.name); temp_id->flag = id.flag; temp_id->session_uid = id.session_uid; if (is_undo) { temp_id->recalc_up_to_undo_push = id.recalc_up_to_undo_push; temp_id->tag = id.tag & ID_TAG_KEEP_ON_UNDO; } return; } /* Regular 'full' ID writing, copy everything, then clear some runtime data irrelevant in the * blendfile. */ memcpy(temp_id, &id, id_type->struct_size); /* Clear runtime data to reduce false detection of changed data in undo/redo context. */ if (is_undo) { temp_id->tag &= ID_TAG_KEEP_ON_UNDO; } else { temp_id->tag = 0; } temp_id->us = 0; temp_id->icon_id = 0; temp_id->runtime = nullptr; /* Those listbase data change every time we add/remove an ID, and also often when * renaming one (due to re-sorting). This avoids generating a lot of false 'is changed' * detections between undo steps. */ temp_id->prev = nullptr; temp_id->next = nullptr; /* Those runtime pointers should never be set during writing stage, but just in case clear * them too. */ temp_id->orig_id = nullptr; temp_id->newid = nullptr; /* Even though in theory we could be able to preserve this python instance across undo even * when we need to re-read the ID into its original address, this is currently cleared in * #direct_link_id_common in `readfile.cc` anyway. */ temp_id->py_instance = nullptr; } BLO_Write_IDBuffer::BLO_Write_IDBuffer(ID &id, BlendWriter *writer) : BLO_Write_IDBuffer(id, BLO_write_is_undo(writer), false) { } static std::string get_blend_file_header() { if (SYSTEM_SUPPORTS_WRITING_FILE_VERSION_1 && !USER_DEVELOPER_TOOL_TEST(&U, write_legacy_blend_file_format)) { const int header_size_in_bytes = SIZEOFBLENDERHEADER_VERSION_1; /* New blend file header format. */ std::stringstream ss; ss << "BLENDER"; ss << header_size_in_bytes; ss << '-'; ss << std::setfill('0') << std::setw(2) << BLEND_FILE_FORMAT_VERSION_1; ss << 'v'; ss << std::setfill('0') << std::setw(4) << BLENDER_FILE_VERSION; const std::string header = ss.str(); BLI_assert(header.size() == header_size_in_bytes); return header; } const char pointer_size_char = sizeof(void *) == 8 ? '-' : '_'; const char endian_char = 'v'; /* Legacy blend file header format. */ std::stringstream ss; ss << "BLENDER"; ss << pointer_size_char; ss << endian_char; ss << BLENDER_FILE_VERSION; const std::string header = ss.str(); BLI_assert(header.size() == SIZEOFBLENDERHEADER_VERSION_0); return header; } static void write_blend_file_header(WriteData *wd) { const std::string header = get_blend_file_header(); mywrite(wd, header.data(), header.size()); } /** * Gathers all local IDs that should be written to the file. */ static Vector gather_local_ids_to_write(Main *bmain, const bool is_undo) { Vector local_ids_to_write; ID *id; FOREACH_MAIN_ID_BEGIN (bmain, id) { if (GS(id->name) == ID_LI) { /* Libraries are handled separately below. */ continue; } if (ID_IS_LINKED(id)) { /* Linked data-blocks are handled separately below. */ continue; } const IDTypeInfo *id_type = BKE_idtype_get_info_from_id(id); UNUSED_VARS_NDEBUG(id_type); /* We should never attempt to write non-regular IDs * (i.e. all kind of temp/runtime ones). */ BLI_assert((id->tag & (ID_TAG_NO_MAIN | ID_TAG_NO_USER_REFCOUNT | ID_TAG_NOT_ALLOCATED)) == 0); /* We only write unused IDs in undo case. */ if (!is_undo) { /* NOTE: All 'never unused' local IDs (Scenes, WindowManagers, ...) should always be * written to disk, so their user-count should never be zero currently. Note that * libraries have already been skipped above, as they need a specific handling. */ if (id->us == 0) { /* FIXME: #124857: Some old files seem to cause incorrect handling of their temp * screens. * * See e.g. file attached to #124777 (from 2.79.1). * * For now ignore, issue is not obvious to track down (`temp` bScreen ID from read data * _does_ have the proper `temp` tag), and seems anecdotal at worst. */ BLI_assert((id_type->flags & IDTYPE_FLAGS_NEVER_UNUSED) == 0); continue; } /* XXX Special handling for ShapeKeys, as having unused shape-keys is not a good thing * (and reported as error by e.g. `BLO_main_validate_shapekeys`), skip writing shape-keys * when their 'owner' is not written. * * NOTE: Since ShapeKeys are conceptually embedded IDs (like root node trees e.g.), this * behavior actually makes sense anyway. This remains more of a temp hack until topic of * how to handle unused data on save is properly tackled. */ if (GS(id->name) == ID_KE) { Key *shape_key = reinterpret_cast(id); /* NOTE: Here we are accessing the real owner ID data, not it's 'proxy' shallow copy * generated for its file-writing. This is not expected to be an issue, but is worth * noting. */ if (shape_key->from == nullptr || shape_key->from->us == 0) { continue; } } } if ((id->tag & ID_TAG_RUNTIME) != 0 && !is_undo) { /* Runtime IDs are never written to .blend files, and they should not influence * (in)direct status of linked IDs they may use. */ continue; } local_ids_to_write.append(id); } FOREACH_MAIN_ID_END; return local_ids_to_write; } /** * Precomputes a stable pointer for each data-block before they are used. This ensures that their * written pointer does not depend on the order in which data-blocks are written. */ static void prepare_stable_data_block_ids(WriteData &wd, Main &bmain) { ID *id; FOREACH_MAIN_ID_BEGIN (&bmain, id) { /* Ensure no other stable pointer has been created before. */ BLI_assert(!wd.stable_address_ids.pointer_map.contains(id)); /* IDs themselves never need to get stable addresses in undo case. */ if (wd.use_memfile) { continue; } /* Derive the stable pointer from the id/library name which is independent of the write-order * of data-blocks. */ uint64_t hint = get_stable_pointer_hint_for_id(*id, wd.use_memfile); const uint64_t address_id = get_next_stable_address_id(wd, hint); /* Store the computed stable pointer so that it is used whenever the data-block is written or * referenced. */ wd.stable_address_ids.pointer_map.add(id, address_id); } FOREACH_MAIN_ID_END; } /** * When #MemFile arguments are non-null, this is a file-safe to memory. * * \param compare: Previous memory file (can be nullptr). * \param current: The current memory file (can be nullptr). */ static bool write_file_handle(Main *mainvar, WriteWrap *ww, MemFile *compare, MemFile *current, const int write_flags, const bool use_userdef, const BlendThumbnail *thumb, std::ostream *debug_dst) { WriteData *wd; wd = mywrite_begin(ww, compare, current); wd->debug_dst = debug_dst; BlendWriter writer = {wd}; BKE_main_view_layers_synced_ensure(mainvar); prepare_stable_data_block_ids(*wd, *mainvar); /* Recompute all ID user-counts if requested. Allows to avoid skipping writing of IDs wrongly * detected as unused due to invalid user-count. */ if (!wd->use_memfile) { if (USER_DEVELOPER_TOOL_TEST(&U, use_recompute_usercount_on_save_debug)) { BKE_main_id_refcount_recompute(mainvar, false); } } write_blend_file_header(wd); write_renderinfo(wd, mainvar); write_thumb(wd, thumb); write_global(wd, write_flags, mainvar); /* The window-manager and screen often change, * avoid thumbnail detecting changes because of this. */ mywrite_flush(wd); const bool is_undo = wd->use_memfile; Vector local_ids_to_write = gather_local_ids_to_write(mainvar, is_undo); if (!is_undo) { BKE_main_id_indirect_linked_update(*mainvar, local_ids_to_write); /* Forcefully ensure we know about all needed override operations. */ for (ID *id : local_ids_to_write) { if (ID_IS_OVERRIDE_LIBRARY_REAL(id) && !ID_IS_OVERRIDE_LIBRARY_VIRTUAL(id)) { BKE_lib_override_library_operations_create(mainvar, id, nullptr); } } } /* Actually write local data-blocks to the file. */ for (ID *id : local_ids_to_write) { write_id(wd, id); } /* Write libraries about libraries and linked data-blocks. */ write_libraries(wd, mainvar); /* So changes above don't cause a 'DNA1' to be detected as changed on undo. */ mywrite_flush(wd); if (use_userdef) { write_userdef(&writer, &U); } /* Write DNA last, because (to be implemented) test for which structs are written. * * Note that we *borrow* the pointer to 'DNAstr', * so writing each time uses the same address and doesn't cause unnecessary undo overhead. */ writedata(wd, BLO_CODE_DNA1, size_t(wd->sdna->data_size), wd->sdna->data); /* End of file. */ BHead bhead{}; bhead.code = BLO_CODE_ENDB; write_bhead(wd, bhead); return mywrite_end(wd); } /** * Do reverse file history: `.blend1` -> `.blend2`, `.blend` -> `.blend1` ... etc. * \return True on success. */ static bool do_history(const char *filepath, ReportList *reports) { /* Add 2 because version number maximum is double-digits. */ char filepath_tmp1[FILE_MAX + 2], filepath_tmp2[FILE_MAX + 2]; int version_number = min_ii(99, U.versions); if (version_number == 0) { return true; } if (strlen(filepath) < 2) { BKE_report(reports, RPT_ERROR, "Unable to make version backup: filename too short"); return false; } while (version_number > 1) { SNPRINTF(filepath_tmp1, "%s%d", filepath, version_number - 1); if (BLI_exists(filepath_tmp1)) { SNPRINTF(filepath_tmp2, "%s%d", filepath, version_number); if (BLI_rename_overwrite(filepath_tmp1, filepath_tmp2)) { BKE_report(reports, RPT_ERROR, "Unable to make version backup"); return false; } } version_number--; } /* Needed when `version_number == 1`. */ if (BLI_exists(filepath)) { SNPRINTF(filepath_tmp1, "%s%d", filepath, version_number); if (BLI_rename_overwrite(filepath, filepath_tmp1)) { BKE_report(reports, RPT_ERROR, "Unable to make version backup"); return false; } } return true; } static void write_file_main_validate_pre(Main &bmain, const BlendFileWriteParams ¶ms, ReportList *reports) { if (!bmain.lock) { return; } if (G.debug & G_DEBUG_IO) { BKE_report( reports, RPT_DEBUG, "Checking validity of current .blend file *BEFORE* save to disk"); } BLO_main_validate_shapekeys(&bmain, reports); if (!BKE_main_namemap_validate_and_fix(bmain)) { BKE_report(reports, RPT_ERROR, "Critical data corruption: Conflicts and/or otherwise invalid data-blocks names " "(see console for details)"); } if (G.debug & G_DEBUG_IO) { BLO_main_validate_libraries(&bmain, reports); } if (!params.is_copypaste_buffer) { bool has_clipboard_flag = false; for (ID &id_iter : MainAllIDsIterator(bmain)) { if (id_iter.flag & ID_FLAG_CLIPBOARD_MARK) { CLOG_WARN(&LOG, "Found an ID '%s' flagged as copy/paste clipboard data while writing a regular " "blendfile, clearing the flag", id_iter.name); id_iter.flag &= ~ID_FLAG_CLIPBOARD_MARK; has_clipboard_flag = true; } } if (has_clipboard_flag) { BKE_report(reports, RPT_INFO, "Found IDs flagged as copy/paste clipboard data while writing a regular " "blendfile, the flag has been cleared in these IDs"); } } } static void write_file_main_validate_post(Main *bmain, ReportList *reports) { if (!bmain->lock) { return; } if (G.debug & G_DEBUG_IO) { BKE_report( reports, RPT_DEBUG, "Checking validity of current .blend file *BEFORE* save to disk"); BLO_main_validate_libraries(bmain, reports); } } static bool BLO_write_file_impl(Main *mainvar, const char *filepath, const int write_flags, const BlendFileWriteParams *params, ReportList *reports, WriteWrap &ww) { BLI_assert(!BLI_path_is_rel(filepath)); BLI_assert(BLI_path_is_abs_from_cwd(filepath)); char tempname[FILE_MAX + 1]; eBLO_WritePathRemap remap_mode = params->remap_mode; const bool use_save_versions = params->use_save_versions; const bool use_save_as_copy = params->use_save_as_copy; const bool use_userdef = params->use_userdef; const BlendThumbnail *thumb = params->thumb; const bool relbase_valid = (mainvar->filepath[0] != '\0'); /* Extra protection: Never save a non asset file as asset file. Otherwise a normal file is turned * into an asset file, which can result in data loss because the asset system will allow editing * this file from the UI, regenerating its content with just the asset and it dependencies. */ if ((write_flags & G_FILE_ASSET_EDIT_FILE) && !mainvar->is_asset_edit_file) { BKE_reportf(reports, RPT_ERROR, "Cannot save normal file (%s) as asset system file", tempname); return false; } /* Path backup/restore. */ void *path_list_backup = nullptr; const eBPathForeachFlag path_list_flag = (BKE_BPATH_FOREACH_PATH_SKIP_LINKED | BKE_BPATH_FOREACH_PATH_SKIP_MULTIFILE); write_file_main_validate_pre(*mainvar, *params, reports); /* Open temporary file, so we preserve the original in case we crash. */ SNPRINTF(tempname, "%s@", filepath); if (ww.open(tempname) == false) { BKE_reportf( reports, RPT_ERROR, "Cannot open file %s for writing: %s", tempname, strerror(errno)); return false; } if (remap_mode == BLO_WRITE_PATH_REMAP_ABSOLUTE) { /* Paths will already be absolute, no remapping to do. */ if (relbase_valid == false) { remap_mode = BLO_WRITE_PATH_REMAP_NONE; } } /* Remapping of relative paths to new file location. */ if (remap_mode != BLO_WRITE_PATH_REMAP_NONE) { if (remap_mode == BLO_WRITE_PATH_REMAP_RELATIVE) { /* Make all relative as none of the existing paths can be relative in an unsaved document. */ if (relbase_valid == false) { remap_mode = BLO_WRITE_PATH_REMAP_RELATIVE_ALL; } } /* The source path only makes sense to set if the file was saved (`relbase_valid`). */ char dir_src[FILE_MAX]; char dir_dst[FILE_MAX]; /* Normalize the paths in case there is some subtle difference (so they can be compared). */ if (relbase_valid) { BLI_path_split_dir_part(mainvar->filepath, dir_src, sizeof(dir_src)); BLI_path_normalize(dir_src); } else { dir_src[0] = '\0'; } BLI_path_split_dir_part(filepath, dir_dst, sizeof(dir_dst)); BLI_path_normalize(dir_dst); /* Only for relative, not relative-all, as this means making existing paths relative. */ if (remap_mode == BLO_WRITE_PATH_REMAP_RELATIVE) { if (relbase_valid && (BLI_path_cmp(dir_dst, dir_src) == 0)) { /* Saved to same path. Nothing to do. */ remap_mode = BLO_WRITE_PATH_REMAP_NONE; } } else if (remap_mode == BLO_WRITE_PATH_REMAP_ABSOLUTE) { if (relbase_valid == false) { /* Unsaved, all paths are absolute.Even if the user manages to set a relative path, * there is no base-path that can be used to make it absolute. */ remap_mode = BLO_WRITE_PATH_REMAP_NONE; } } if (remap_mode != BLO_WRITE_PATH_REMAP_NONE) { /* Some path processing (e.g. with libraries) may use the current `main->filepath`, if this * is not matching the path currently used for saving, unexpected paths corruptions can * happen. See #98201. */ char mainvar_filepath_orig[FILE_MAX]; STRNCPY(mainvar_filepath_orig, mainvar->filepath); STRNCPY(mainvar->filepath, filepath); /* Check if we need to backup and restore paths. */ if (UNLIKELY(use_save_as_copy)) { path_list_backup = BKE_bpath_list_backup(mainvar, path_list_flag); } switch (remap_mode) { case BLO_WRITE_PATH_REMAP_RELATIVE: /* Saved, make relative paths relative to new location (if possible). */ BLI_assert(relbase_valid); BKE_bpath_relative_rebase(mainvar, dir_src, dir_dst, nullptr); break; case BLO_WRITE_PATH_REMAP_RELATIVE_ALL: /* If saved, make relative paths relative to new location (if possible). */ if (relbase_valid && (BLI_path_cmp(dir_dst, dir_src) != 0)) { BKE_bpath_relative_rebase(mainvar, dir_src, dir_dst, nullptr); } /* Make all absolute paths relative (when requested or unsaved). */ BKE_bpath_relative_convert(mainvar, dir_dst, nullptr); break; case BLO_WRITE_PATH_REMAP_ABSOLUTE: /* Make all absolute (when requested or unsaved). */ BLI_assert(relbase_valid); BKE_bpath_absolute_convert(mainvar, dir_src, nullptr); break; case BLO_WRITE_PATH_REMAP_NONE: BLI_assert_unreachable(); /* Unreachable. */ break; } STRNCPY(mainvar->filepath, mainvar_filepath_orig); } } #if GENERATE_DEBUG_BLEND_FILE std::string debug_dst_path = StringRef(filepath) + DEBUG_BLEND_FILE_SUFFIX; fstream debug_dst_file(debug_dst_path, std::ios::out); std::ostream *debug_dst = &debug_dst_file; #else std::ostream *debug_dst = nullptr; #endif /* Actual file writing. */ const bool err = write_file_handle( mainvar, &ww, nullptr, nullptr, write_flags, use_userdef, thumb, debug_dst); ww.close(); if (UNLIKELY(path_list_backup)) { BKE_bpath_list_restore(mainvar, path_list_flag, path_list_backup); BKE_bpath_list_free(path_list_backup); } if (err) { BKE_report(reports, RPT_ERROR, strerror(errno)); remove(tempname); return false; } /* File save to temporary file was successful, now do reverse file history * (move `.blend1` -> `.blend2`, `.blend` -> `.blend1` .. etc). */ if (use_save_versions) { if (!do_history(filepath, reports)) { BKE_report(reports, RPT_ERROR, "Version backup failed (file saved with @)"); return false; } } if (BLI_rename_overwrite(tempname, filepath) != 0) { BKE_report(reports, RPT_ERROR, "Cannot change old file (file saved with @)"); return false; } write_file_main_validate_post(mainvar, reports); if (mainvar->is_global_main && !params->use_save_as_copy) { /* It is used to reload Blender after a crash on Windows OS. */ STRNCPY(G.filepath_last_blend, filepath); } return true; } /** \} */ /* -------------------------------------------------------------------- */ /** \name File Writing (Public) * \{ */ bool BLO_write_file(Main *mainvar, const char *filepath, const int write_flags, const BlendFileWriteParams *params, ReportList *reports) { RawWriteWrap raw_wrap; if (write_flags & G_FILE_COMPRESS) { ZstdWriteWrap zstd_wrap(raw_wrap); return BLO_write_file_impl(mainvar, filepath, write_flags, params, reports, zstd_wrap); } return BLO_write_file_impl(mainvar, filepath, write_flags, params, reports, raw_wrap); } bool BLO_write_file_mem(Main *mainvar, MemFile *compare, MemFile *current, const int write_flags) { bool use_userdef = false; const bool err = write_file_handle( mainvar, nullptr, compare, current, write_flags, use_userdef, nullptr, nullptr); return (err == 0); } bool BLO_write_file_to_callback(Main *mainvar, const int write_flags, BLO_WriteFileCallback callback, void *user_data) { if (callback == nullptr) { return false; } CallbackWriteWrap callback_wrap(callback, user_data); const bool err = write_file_handle( mainvar, &callback_wrap, nullptr, nullptr, write_flags, false, nullptr, nullptr); return (err == 0); } /* * API to write chunks of data. */ void BlendWriter::write_struct_by_name(const char *struct_name, const void *data) { this->write_struct_array_by_name(struct_name, 1, data); } void BlendWriter::write_struct_array_by_name(const char *struct_name, const int64_t array_size, const void *data) { int struct_id = this->struct_id_by_name(struct_name); if (UNLIKELY(struct_id == -1)) { CLOG_ERROR(&LOG, "Can't find SDNA code <%s>", struct_name); return; } this->write_struct_array_by_id(struct_id, array_size, data); } void BlendWriter::write_struct_by_id(const int struct_id, const void *data) { writestruct_nr(this->wd, BLO_CODE_DATA, struct_id, 1, data); } void BlendWriter::write_struct_at_address_by_id(const int struct_id, const void *address, const void *data) { this->write_struct_at_address_by_id_with_filecode(BLO_CODE_DATA, struct_id, address, data); } void BlendWriter::write_struct_at_address_by_id_with_filecode(const int filecode, const int struct_id, const void *address, const void *data) { writestruct_at_address_nr(this->wd, filecode, struct_id, 1, address, data); } void BlendWriter::write_struct_array_by_id(const int struct_id, const int64_t array_size, const void *data) { writestruct_nr(this->wd, BLO_CODE_DATA, struct_id, array_size, data); } void BlendWriter::write_struct_array_at_address_by_id(const int struct_id, const int64_t array_size, const void *address, const void *data) { writestruct_at_address_nr(this->wd, BLO_CODE_DATA, struct_id, array_size, address, data); } void BlendWriter::write_struct_list_by_id(const int struct_id, const ListBase *list) { writelist_nr(this->wd, BLO_CODE_DATA, struct_id, list); } void BlendWriter::write_struct_list_by_name(const char *struct_name, ListBase *list) { int struct_id = this->struct_id_by_name(struct_name); if (UNLIKELY(struct_id == -1)) { CLOG_ERROR(&LOG, "Can't find SDNA code <%s>", struct_name); return; } this->write_struct_list_by_id(struct_id, list); } int BlendWriter::struct_id_by_name(const char *struct_name) const { int struct_id = DNA_struct_find_with_alias(this->wd->sdna, struct_name); return struct_id; } void BlendWriter::write_raw(size_t size_in_bytes, const void *data) { writedata(this->wd, BLO_CODE_DATA, size_in_bytes, data); } void BlendWriter::write_char_array(int64_t num, const char *data) { this->write_raw(sizeof(char) * size_t(num), data); } void BlendWriter::write_int8_array(int64_t num, const int8_t *data) { this->write_raw(sizeof(int8_t) * size_t(num), data); } void BlendWriter::write_int16_array(int64_t num, const int16_t *data) { this->write_raw(sizeof(int16_t) * size_t(num), data); } void BlendWriter::write_uint8_array(int64_t num, const uint8_t *data) { this->write_raw(sizeof(uint8_t) * size_t(num), data); } void BlendWriter::write_int32_array(int64_t num, const int32_t *data) { this->write_raw(sizeof(int32_t) * size_t(num), data); } void BlendWriter::write_uint32_array(int64_t num, const uint32_t *data) { this->write_raw(sizeof(uint32_t) * size_t(num), data); } void BlendWriter::write_float_array(int64_t num, const float *data) { this->write_raw(sizeof(float) * size_t(num), data); } void BlendWriter::write_double_array(int64_t num, const double *data) { this->write_raw(sizeof(double) * size_t(num), data); } void BlendWriter::write_float3_array(int64_t num, const float *data) { this->write_raw(sizeof(float[3]) * size_t(num), data); } void BlendWriter::write_pointer_array(int64_t num, const void *data_ptr) { /* Create a temporary copy of the pointer array, because all pointers need to be remapped to * their stable address ids. */ Array data = Span( reinterpret_cast(data_ptr), num); for (const int64_t i : data.index_range()) { data[i] = get_address_id(*this->wd, data[i]); } writedata(this->wd, BLO_CODE_DATA, data.data(), data.as_span().size_in_bytes(), data_ptr); } void BlendWriter::write_string(const char *data) { if (data != nullptr) { this->write_raw(strlen(data) + 1, data); } } void BLO_write_generated_pointer_tag(BlendWriter *writer, const void *data) { if (!data) { return; } if (!BLO_write_is_undo(writer)) { return; } const uint64_t address_id = get_address_id_for_undo(*writer->wd); /* Check that the pointer has not been written before it was tagged as being generated. */ BLI_assert(writer->wd->stable_address_ids.pointer_map.lookup_default(data, address_id) == address_id); writer->wd->stable_address_ids.pointer_map.add(data, address_id); } void BLO_write_shared(BlendWriter *writer, const void *data, const size_t approximate_size_in_bytes, const ImplicitSharingInfo *sharing_info, const FunctionRef write_fn) { if (data == nullptr) { return; } const uint64_t address_id = get_address_id_int(*writer->wd, data); if (BLO_write_is_undo(writer)) { MemFile &memfile = *writer->wd->mem.written_memfile; if (sharing_info != nullptr) { if (memfile.shared_storage == nullptr) { memfile.shared_storage = MEM_new(__func__); } if (memfile.shared_storage->sharing_info_by_address_id.add(address_id, {sharing_info, data})) { /* The undo-step takes (shared) ownership of the data, which also makes it immutable. */ sharing_info->add_user(); /* This size is an estimate, but good enough to count data with many users less. */ memfile.size += approximate_size_in_bytes / sharing_info->strong_users(); return; } } } if (sharing_info != nullptr) { if (!writer->wd->per_id_written_shared_addresses.add(data)) { /* Was written already. */ return; } } write_fn(); } bool BLO_write_is_undo(BlendWriter *writer) { return writer->wd->use_memfile; } /** \} */ } // namespace blender