From 1a62fdc82a6264583af8673b099b37c874b899bd Mon Sep 17 00:00:00 2001 From: Devashish Lal Date: Mon, 10 Feb 2025 16:56:52 +0100 Subject: [PATCH] Geometry Nodes: CSV import node This commit implements a node to import CSV files as a point cloud. The interface is minimal, with just a file path input. The type of each column is chosen by whether the first value is an integer or a float (those are currently the only supported types). The goal of the node is to make it easier to get arbitrary data into geometry nodes for visualization purposes, for example. https://devtalk.blender.org/t/gsoc-2024-geometry-nodes-file-import-nodes/34482 Pull Request: https://projects.blender.org/blender/blender/pulls/126308 --- CMakeLists.txt | 3 + build_files/cmake/config/blender_lite.cmake | 1 + .../startup/bl_ui/node_add_menu_geometry.py | 1 + source/blender/io/CMakeLists.txt | 6 +- source/blender/io/common/CMakeLists.txt | 4 + .../IO_string_utils.hh} | 38 ++- .../intern/string_utils.cc} | 63 +++- .../intern/string_utils_tests.cc} | 20 +- source/blender/io/csv/CMakeLists.txt | 40 +++ source/blender/io/csv/IO_csv.cc | 22 ++ source/blender/io/csv/IO_csv.hh | 27 ++ source/blender/io/csv/importer/csv_data.cc | 61 ++++ source/blender/io/csv/importer/csv_data.hh | 54 ++++ source/blender/io/csv/importer/csv_reader.cc | 268 ++++++++++++++++++ source/blender/io/csv/importer/csv_reader.hh | 18 ++ .../blender/io/wavefront_obj/CMakeLists.txt | 3 - .../importer/obj_import_file_reader.cc | 3 +- .../blender/makesrna/intern/rna_nodetree.cc | 1 + source/blender/nodes/geometry/CMakeLists.txt | 12 +- .../geometry/nodes/node_geo_import_csv.cc | 80 ++++++ 20 files changed, 700 insertions(+), 25 deletions(-) rename source/blender/io/{wavefront_obj/importer/obj_import_string_utils.hh => common/IO_string_utils.hh} (68%) rename source/blender/io/{wavefront_obj/importer/obj_import_string_utils.cc => common/intern/string_utils.cc} (70%) rename source/blender/io/{wavefront_obj/tests/obj_import_string_utils_tests.cc => common/intern/string_utils_tests.cc} (90%) create mode 100644 source/blender/io/csv/CMakeLists.txt create mode 100644 source/blender/io/csv/IO_csv.cc create mode 100644 source/blender/io/csv/IO_csv.hh create mode 100644 source/blender/io/csv/importer/csv_data.cc create mode 100644 source/blender/io/csv/importer/csv_data.hh create mode 100644 source/blender/io/csv/importer/csv_reader.cc create mode 100644 source/blender/io/csv/importer/csv_reader.hh create mode 100644 source/blender/nodes/geometry/nodes/node_geo_import_csv.cc diff --git a/CMakeLists.txt b/CMakeLists.txt index da941b7e1b5..554b6ec401c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -486,6 +486,9 @@ option(WITH_IO_PLY "Enable PLY 3D file format support (*.ply)" ON) option(WITH_IO_STL "Enable STL 3D file format support (*.stl)" ON) option(WITH_IO_GREASE_PENCIL "Enable grease-pencil file format IO (*.svg, *.pdf)" ON) +# Csv support +option(WITH_IO_CSV "Enable CSV file format support (*.csv)" ON) + # Sound output option(WITH_SDL "Enable SDL for sound" OFF) option(WITH_OPENAL "Enable OpenAL Support (http://www.openal.org)" ON) diff --git a/build_files/cmake/config/blender_lite.cmake b/build_files/cmake/config/blender_lite.cmake index 9b1522b66e5..feebfa2937d 100644 --- a/build_files/cmake/config/blender_lite.cmake +++ b/build_files/cmake/config/blender_lite.cmake @@ -35,6 +35,7 @@ set(WITH_INPUT_NDOF OFF CACHE BOOL "" FORCE) set(WITH_INTERNATIONAL OFF CACHE BOOL "" FORCE) set(WITH_IO_PLY OFF CACHE BOOL "" FORCE) set(WITH_IO_STL OFF CACHE BOOL "" FORCE) +set(WITH_IO_CSV OFF CACHE BOOL "" FORCE) set(WITH_IO_WAVEFRONT_OBJ OFF CACHE BOOL "" FORCE) set(WITH_IO_GREASE_PENCIL OFF CACHE BOOL "" FORCE) set(WITH_JACK OFF CACHE BOOL "" FORCE) diff --git a/scripts/startup/bl_ui/node_add_menu_geometry.py b/scripts/startup/bl_ui/node_add_menu_geometry.py index d7f7b1618c6..6ac082c58f5 100644 --- a/scripts/startup/bl_ui/node_add_menu_geometry.py +++ b/scripts/startup/bl_ui/node_add_menu_geometry.py @@ -475,6 +475,7 @@ class NODE_MT_category_import(Menu): def draw(self, _context): layout = self.layout + node_add_menu.add_node_type(layout, "GeometryNodeImportCSV") node_add_menu.add_node_type(layout, "GeometryNodeImportOBJ") node_add_menu.add_node_type(layout, "GeometryNodeImportPLY") node_add_menu.add_node_type(layout, "GeometryNodeImportSTL") diff --git a/source/blender/io/CMakeLists.txt b/source/blender/io/CMakeLists.txt index f2045767187..cbb20aee749 100644 --- a/source/blender/io/CMakeLists.txt +++ b/source/blender/io/CMakeLists.txt @@ -2,7 +2,7 @@ # # SPDX-License-Identifier: GPL-2.0-or-later -if(WITH_IO_WAVEFRONT_OBJ OR WITH_IO_PLY OR WITH_IO_STL OR WITH_IO_GREASE_PENCIL OR WITH_ALEMBIC OR WITH_USD) +if(WITH_IO_WAVEFRONT_OBJ OR WITH_IO_PLY OR WITH_IO_STL OR WITH_IO_GREASE_PENCIL OR WITH_ALEMBIC OR WITH_USD OR WITH_IO_CSV) add_subdirectory(common) endif() @@ -33,3 +33,7 @@ endif() if(WITH_USD) add_subdirectory(usd) endif() + +if (WITH_IO_CSV) + add_subdirectory(csv) +endif() diff --git a/source/blender/io/common/CMakeLists.txt b/source/blender/io/common/CMakeLists.txt index bb07d9f510e..972211c8df3 100644 --- a/source/blender/io/common/CMakeLists.txt +++ b/source/blender/io/common/CMakeLists.txt @@ -8,6 +8,7 @@ set(INC ) set(INC_SYS + ../../../../extern/fast_float ) set(SRC @@ -18,6 +19,7 @@ set(SRC intern/orientation.cc intern/path_util.cc intern/subdiv_disabler.cc + intern/string_utils.cc IO_abstract_hierarchy_iterator.h IO_dupli_persistent_id.hh @@ -26,6 +28,7 @@ set(SRC IO_path_util_types.hh IO_subdiv_disabler.hh IO_types.hh + IO_string_utils.hh intern/dupli_parent_finder.hh ) @@ -48,6 +51,7 @@ if(WITH_GTESTS) intern/abstract_hierarchy_iterator_test.cc intern/hierarchy_context_order_test.cc intern/object_identifier_test.cc + intern/string_utils_tests.cc ) set(TEST_INC ../../blenloader diff --git a/source/blender/io/wavefront_obj/importer/obj_import_string_utils.hh b/source/blender/io/common/IO_string_utils.hh similarity index 68% rename from source/blender/io/wavefront_obj/importer/obj_import_string_utils.hh rename to source/blender/io/common/IO_string_utils.hh index efb5e49c752..114d6898184 100644 --- a/source/blender/io/wavefront_obj/importer/obj_import_string_utils.hh +++ b/source/blender/io/common/IO_string_utils.hh @@ -1,4 +1,4 @@ -/* SPDX-FileCopyrightText: 2023 Blender Authors +/* SPDX-FileCopyrightText: 2024 Blender Authors * * SPDX-License-Identifier: GPL-2.0-or-later */ @@ -7,18 +7,18 @@ #include "BLI_string_ref.hh" /* - * Various text parsing utilities used by OBJ importer. + * Various text parsing utilities used by importers. * * Many of these functions take two pointers (p, end) indicating * which part of a string to operate on, and return a possibly * changed new start of the string. They could be taking a StringRef * as input and returning a new StringRef, but this is a hot path - * in OBJ parsing, and the StringRef approach does lose performance + * in CSV and OBJ parsing, and the StringRef approach does lose performance * (mostly due to return of StringRef being two register-size values * instead of just one pointer). */ -namespace blender::io::obj { +namespace blender::io { /** * Fetches next line from an input string buffer. @@ -45,6 +45,34 @@ const char *drop_whitespace(const char *p, const char *end); */ const char *drop_non_whitespace(const char *p, const char *end); +/** + * Parse an integer from an input string. + * The parsed result is stored in `dst`. The function skips + * leading white-space unless `skip_space=false`. If the + * number can't be parsed (invalid syntax, out of range), + * `success` value is false. + * + * Returns the start of remainder of the input string after parsing. + */ +const char *try_parse_int( + const char *p, const char *end, int fallback, bool &success, int &dst, bool skip_space = true); + +/** + * Parse a float from an input string. + * The parsed result is stored in `dst`. The function skips + * leading white-space unless `skip_space=false`. If the + * number can't be parsed (invalid syntax, out of range), + * `success` value is false. + * + * Returns the start of remainder of the input string after parsing. + */ +const char *try_parse_float(const char *p, + const char *end, + int fallback, + bool &success, + float &dst, + bool skip_space = true); + /** * Parse an integer from an input string. * The parsed result is stored in `dst`. The function skips @@ -89,4 +117,4 @@ const char *parse_floats(const char *p, int count, bool require_trailing_space = false); -} // namespace blender::io::obj +} // namespace blender::io diff --git a/source/blender/io/wavefront_obj/importer/obj_import_string_utils.cc b/source/blender/io/common/intern/string_utils.cc similarity index 70% rename from source/blender/io/wavefront_obj/importer/obj_import_string_utils.cc rename to source/blender/io/common/intern/string_utils.cc index 0ce80e8e0fe..49263eb817d 100644 --- a/source/blender/io/wavefront_obj/importer/obj_import_string_utils.cc +++ b/source/blender/io/common/intern/string_utils.cc @@ -1,8 +1,8 @@ -/* SPDX-FileCopyrightText: 2023 Blender Authors +/* SPDX-FileCopyrightText: 2024 Blender Authors * * SPDX-License-Identifier: GPL-2.0-or-later */ -#include "obj_import_string_utils.hh" +#include "IO_string_utils.hh" /* NOTE: we could use C++17 from_chars to parse * floats, but even if some compilers claim full support, @@ -13,7 +13,7 @@ #include "fast_float.h" #include -namespace blender::io::obj { +namespace blender::io { StringRef read_next_line(StringRef &buffer) { @@ -76,6 +76,61 @@ const char *drop_non_whitespace(const char *p, const char *end) return p; } +static const char *drop_sign(const char *p, const char *end, int &sign) +{ + sign = 1; + if (p < end) { + if (*p == '+') { + ++p; + } + if (*p == '-') { + sign = -1; + ++p; + } + } + return p; +} + +const char *try_parse_float( + const char *p, const char *end, int fallback, bool &success, float &dst, bool skip_space) +{ + if (skip_space) { + p = drop_whitespace(p, end); + } + int sign = 0; + p = drop_sign(p, end, sign); + fast_float::from_chars_result res = fast_float::from_chars(p, end, dst); + if (ELEM(res.ec, std::errc::invalid_argument, std::errc::result_out_of_range) || res.ptr < end) { + dst = fallback; + success = false; + } + else { + dst *= sign; + success = true; + } + return res.ptr; +} + +const char *try_parse_int( + const char *p, const char *end, int fallback, bool &success, int &dst, bool skip_space) +{ + if (skip_space) { + p = drop_whitespace(p, end); + } + int sign = 0; + p = drop_sign(p, end, sign); + std::from_chars_result res = std::from_chars(p, end, dst); + if (ELEM(res.ec, std::errc::invalid_argument, std::errc::result_out_of_range) || res.ptr < end) { + dst = fallback; + success = false; + } + else { + dst *= sign; + success = true; + } + return res.ptr; +} + static const char *drop_plus(const char *p, const char *end) { if (p < end && *p == '+') { @@ -133,4 +188,4 @@ const char *parse_int(const char *p, const char *end, int fallback, int &dst, bo return res.ptr; } -} // namespace blender::io::obj +} // namespace blender::io diff --git a/source/blender/io/wavefront_obj/tests/obj_import_string_utils_tests.cc b/source/blender/io/common/intern/string_utils_tests.cc similarity index 90% rename from source/blender/io/wavefront_obj/tests/obj_import_string_utils_tests.cc rename to source/blender/io/common/intern/string_utils_tests.cc index fc38ed19446..4e6748bd21e 100644 --- a/source/blender/io/wavefront_obj/tests/obj_import_string_utils_tests.cc +++ b/source/blender/io/common/intern/string_utils_tests.cc @@ -2,15 +2,15 @@ * * SPDX-License-Identifier: Apache-2.0 */ -#include "obj_import_string_utils.hh" +#include "IO_string_utils.hh" #include "testing/testing.h" -namespace blender::io::obj { +namespace blender::io { #define EXPECT_STRREF_EQ(str1, str2) EXPECT_STREQ(str1, std::string(str2).c_str()) -TEST(obj_import_string_utils, read_next_line) +TEST(io_common_string_utils, read_next_line) { std::string str = "abc\n \n\nline with \t spaces\nCRLF ending:\r\na"; StringRef s = str; @@ -23,7 +23,7 @@ TEST(obj_import_string_utils, read_next_line) EXPECT_TRUE(s.is_empty()); } -TEST(obj_import_string_utils, fixup_line_continuations) +TEST(io_common_string_utils, fixup_line_continuations) { const char *str = "backslash \\\n eol\n" @@ -58,7 +58,7 @@ static StringRef parse_float(StringRef s, parse_float(s.begin(), s.end(), fallback, dst, skip_space, require_trailing_space), s.end()); } -TEST(obj_import_string_utils, drop_whitespace) +TEST(io_common_string_utils, drop_whitespace) { /* Empty */ EXPECT_STRREF_EQ("", drop_whitespace("")); @@ -76,7 +76,7 @@ TEST(obj_import_string_utils, drop_whitespace) EXPECT_STRREF_EQ("d", drop_whitespace(" \t d")); } -TEST(obj_import_string_utils, parse_int_valid) +TEST(io_common_string_utils, parse_int_valid) { std::string str = "1 -10 \t 1234 1234567890 +7 123a"; StringRef s = str; @@ -96,7 +96,7 @@ TEST(obj_import_string_utils, parse_int_valid) EXPECT_STRREF_EQ("a", s); } -TEST(obj_import_string_utils, parse_int_invalid) +TEST(io_common_string_utils, parse_int_invalid) { int val; /* Invalid syntax */ @@ -112,7 +112,7 @@ TEST(obj_import_string_utils, parse_int_invalid) EXPECT_EQ(val, -4); } -TEST(obj_import_string_utils, parse_float_valid) +TEST(io_common_string_utils, parse_float_valid) { std::string str = "1 -10 123.5 -17.125 0.1 1e6 50.0e-1"; StringRef s = str; @@ -134,7 +134,7 @@ TEST(obj_import_string_utils, parse_float_valid) EXPECT_TRUE(s.is_empty()); } -TEST(obj_import_string_utils, parse_float_invalid) +TEST(io_common_string_utils, parse_float_invalid) { float val; /* Invalid syntax */ @@ -153,4 +153,4 @@ TEST(obj_import_string_utils, parse_float_invalid) EXPECT_EQ(val, -5.0f); } -} // namespace blender::io::obj +} // namespace blender::io diff --git a/source/blender/io/csv/CMakeLists.txt b/source/blender/io/csv/CMakeLists.txt new file mode 100644 index 00000000000..1c6c34ceb14 --- /dev/null +++ b/source/blender/io/csv/CMakeLists.txt @@ -0,0 +1,40 @@ +# SPDX-FileCopyrightText: 2023 Blender Authors +# +# SPDX-License-Identifier: GPL-2.0-or-later + +set(INC + . + importer + ../common + ../../blenkernel + ../../bmesh + ../../editors/include + ../../makesrna + ../../windowmanager +) + +set(INC_SYS + ../../../../extern/fast_float +) + +set(SRC + IO_csv.cc + importer/csv_data.cc + importer/csv_reader.cc + + IO_csv.hh + importer/csv_data.hh + importer/csv_reader.hh +) + +set(LIB + bf_blenkernel + PRIVATE bf::blenlib + PRIVATE bf::depsgraph + PRIVATE bf::dna + PRIVATE bf::intern::guardedalloc + bf_io_common + PRIVATE bf::extern::fmtlib +) + +blender_add_lib(bf_io_csv "${SRC}" "${INC}" "${INC_SYS}" "${LIB}") diff --git a/source/blender/io/csv/IO_csv.cc b/source/blender/io/csv/IO_csv.cc new file mode 100644 index 00000000000..03858949839 --- /dev/null +++ b/source/blender/io/csv/IO_csv.cc @@ -0,0 +1,22 @@ +/* SPDX-FileCopyrightText: 2024 Blender Authors + * + * SPDX-License-Identifier: GPL-2.0-or-later */ + +/** \file + * \ingroup csv + */ + +#include "BLI_timeit.hh" + +#include "IO_csv.hh" + +#include "csv_reader.hh" + +namespace blender::io::csv { + +PointCloud *import_csv_as_point_cloud(const CSVImportParams *import_params) +{ + return read_csv_file(*import_params); +} + +} // namespace blender::io::csv diff --git a/source/blender/io/csv/IO_csv.hh b/source/blender/io/csv/IO_csv.hh new file mode 100644 index 00000000000..4eee3ec59e9 --- /dev/null +++ b/source/blender/io/csv/IO_csv.hh @@ -0,0 +1,27 @@ +/* SPDX-FileCopyrightText: 2024 Blender Authors + * + * SPDX-License-Identifier: GPL-2.0-or-later */ + +/** \file + * \ingroup csv + */ + +#pragma once + +#include "BLI_path_utils.hh" + +struct PointCloud; +struct ReportList; + +namespace blender::io::csv { + +struct CSVImportParams { + /** Full path to the source CSV file to import. */ + char filepath[FILE_MAX]; + + ReportList *reports = nullptr; +}; + +PointCloud *import_csv_as_point_cloud(const CSVImportParams *import_params); + +} // namespace blender::io::csv diff --git a/source/blender/io/csv/importer/csv_data.cc b/source/blender/io/csv/importer/csv_data.cc new file mode 100644 index 00000000000..168212a9ca4 --- /dev/null +++ b/source/blender/io/csv/importer/csv_data.cc @@ -0,0 +1,61 @@ +/* SPDX-FileCopyrightText: 2024 Blender Authors + * + * SPDX-License-Identifier: GPL-2.0-or-later */ + +/** \file + * \ingroup csv + */ + +#include "BKE_attribute.hh" +#include "BKE_customdata.hh" +#include "BKE_pointcloud.hh" + +#include "BLI_array_utils.hh" + +#include "csv_data.hh" + +namespace blender::io::csv { + +CsvData::CsvData(const int64_t rows_num, + const Span column_names, + const Span column_types) + : data(column_names.size()), + rows_num(rows_num), + columns_num(column_names.size()), + column_names(column_names), + column_types(column_types) +{ + for (const int i : IndexRange(this->columns_num)) { + data[i] = GArray(*bke::custom_data_type_to_cpp_type(this->column_types[i]), rows_num); + } +} + +PointCloud *CsvData::to_point_cloud() const +{ + PointCloud *point_cloud = BKE_pointcloud_new_nomain(rows_num); + + /* Set all positions to be zero */ + point_cloud->positions_for_write().fill(float3(0.0f, 0.0f, 0.0f)); + + /* Fill the attributes */ + for (const int i : IndexRange(columns_num)) { + const StringRef column_name = column_names[i]; + const eCustomDataType column_type = column_types[i]; + + const CPPType *cpp_column_type = bke::custom_data_type_to_cpp_type(column_type); + GMutableSpan column_data{*cpp_column_type, + MEM_mallocN_aligned(rows_num * cpp_column_type->size(), + cpp_column_type->alignment(), + __func__), + rows_num * cpp_column_type->size()}; + + array_utils::copy(GVArray::ForSpan(data[i]), column_data); + + CustomData_add_layer_named_with_data( + &point_cloud->pdata, column_type, column_data.data(), rows_num, column_name, nullptr); + } + + return point_cloud; +} + +} // namespace blender::io::csv diff --git a/source/blender/io/csv/importer/csv_data.hh b/source/blender/io/csv/importer/csv_data.hh new file mode 100644 index 00000000000..8fc02e8bce8 --- /dev/null +++ b/source/blender/io/csv/importer/csv_data.hh @@ -0,0 +1,54 @@ +/* SPDX-FileCopyrightText: 2024 Blender Authors + * + * SPDX-License-Identifier: GPL-2.0-or-later */ + +/** \file + * \ingroup csv + */ + +#pragma once + +#include + +#include "BLI_array.hh" +#include "BLI_generic_array.hh" +#include "BLI_map.hh" + +struct PointCloud; + +namespace blender::io::csv { + +class CsvData { + private: + Array> data; + + int64_t rows_num; + int64_t columns_num; + + Array column_names; + Array column_types; + + public: + CsvData(int64_t rows_num, Span column_names, Span column_types); + + PointCloud *to_point_cloud() const; + + template void set_data(int64_t row_index, int64_t col_index, const T value) + { + GMutableSpan mutable_span = data[col_index].as_mutable_span(); + MutableSpan typed_mutable_span = mutable_span.typed(); + typed_mutable_span[row_index] = value; + } + + eCustomDataType get_column_type(int64_t col_index) const + { + return column_types[col_index]; + } + + StringRef get_column_name(int64_t col_index) const + { + return column_names[col_index]; + } +}; + +} // namespace blender::io::csv diff --git a/source/blender/io/csv/importer/csv_reader.cc b/source/blender/io/csv/importer/csv_reader.cc new file mode 100644 index 00000000000..d81c8f2c1bf --- /dev/null +++ b/source/blender/io/csv/importer/csv_reader.cc @@ -0,0 +1,268 @@ +/* SPDX-FileCopyrightText: 2024 Blender Authors + * + * SPDX-License-Identifier: GPL-2.0-or-later */ + +/** \file + * \ingroup csv + */ + +#include "BKE_report.hh" + +#include "BLI_fileops.hh" + +#include "IO_csv.hh" +#include "IO_string_utils.hh" + +#include "csv_data.hh" + +#include "csv_reader.hh" + +namespace blender::io::csv { + +static Vector get_columns(const StringRef line) +{ + Vector columns; + const char delim = ','; + const char *start = line.begin(), *end = line.end(); + const char *cell_start = start, *cell_end = start; + + int64_t delim_index = line.find_first_of(delim); + + while (delim_index != StringRef::not_found) { + cell_end = start + delim_index; + + columns.append(std::string(cell_start, cell_end)); + + cell_start = cell_end + 1; + delim_index = line.find_first_of(delim, delim_index + 1); + } + + /* Handle last cell, --end because the end in StringRef is one_after_ern */ + columns.append(std::string(cell_start, --end)); + + return columns; +} + +static std::optional get_column_type(const char *start, const char *end) +{ + bool success = false; + + int _val_int = 0; + try_parse_int(start, end, 0, success, _val_int); + + if (success) { + return CD_PROP_INT32; + } + + float _val_float = 0.0f; + try_parse_float(start, end, 0.0f, success, _val_float); + + if (success) { + return CD_PROP_FLOAT; + } + + return std::nullopt; +} + +static bool get_column_types(const StringRef line, Vector &column_types) +{ + const char delim = ','; + const char *start = line.begin(), *end = line.end(); + const char *cell_start = start, *cell_end = start; + + int64_t delim_index = line.find_first_of(delim); + + while (delim_index != StringRef::not_found) { + cell_end = start + delim_index; + + std::optional column_type = get_column_type(cell_start, cell_end); + if (!column_type.has_value()) { + return false; + } + column_types.append(column_type.value()); + + cell_start = cell_end + 1; + delim_index = line.find_first_of(delim, delim_index + 1); + } + + /* Handle last cell, --end because the end in StringRef is one_after_ern */ + std::optional column_type = get_column_type(cell_start, --end); + if (!column_type.has_value()) { + return false; + } + column_types.append(column_type.value()); + + return true; +} + +static int64_t get_row_count(StringRef buffer) +{ + int64_t row_count = 1; + + while (!buffer.is_empty()) { + read_next_line(buffer); + row_count++; + } + + return row_count; +} + +static void parse_csv_cell(CsvData &csv_data, + int64_t row_index, + int64_t col_index, + const char *start, + const char *end, + const CSVImportParams &import_params) +{ + bool success = false; + + switch (csv_data.get_column_type(col_index)) { + case CD_PROP_INT32: { + int value = 0; + try_parse_int(start, end, 0, success, value); + csv_data.set_data(row_index, col_index, value); + if (!success) { + std::string column_name = csv_data.get_column_name(col_index); + BKE_reportf(import_params.reports, + RPT_ERROR, + "CSV Import: file '%s' has an unexpected value at row %lld for column %s of " + "type Integer", + import_params.filepath, + row_index, + column_name.c_str()); + } + break; + } + case CD_PROP_FLOAT: { + float value = 0.0f; + try_parse_float(start, end, 0.0f, success, value); + csv_data.set_data(row_index, col_index, value); + if (!success) { + std::string column_name = csv_data.get_column_name(col_index); + BKE_reportf(import_params.reports, + RPT_ERROR, + "CSV Import: file '%s' has an unexpected value at row %lld for column %s of " + "type Float", + import_params.filepath, + row_index, + column_name.c_str()); + } + break; + } + default: { + std::string column_name = csv_data.get_column_name(col_index); + BKE_reportf(import_params.reports, + RPT_ERROR, + "CSV Import: file '%s' has an unsupported value at row %lld for column %s", + import_params.filepath, + row_index, + column_name.c_str()); + break; + } + } +} + +static void parse_csv_line(CsvData &csv_data, + int64_t row_index, + const StringRef line, + const CSVImportParams &import_params) +{ + const char delim = ','; + const char *start = line.begin(), *end = line.end(); + const char *cell_start = start, *cell_end = start; + + int64_t col_index = 0; + + int64_t delim_index = line.find_first_of(delim); + + while (delim_index != StringRef::not_found) { + cell_end = start + delim_index; + + parse_csv_cell(csv_data, row_index, col_index, cell_start, cell_end, import_params); + col_index++; + + cell_start = cell_end + 1; + delim_index = line.find_first_of(delim, delim_index + 1); + } + + /* Handle last cell, --end because the end in StringRef is one_after_ern */ + parse_csv_cell(csv_data, row_index, col_index, cell_start, --end, import_params); +} + +static void parse_csv_data(CsvData &csv_data, + StringRef buffer, + const CSVImportParams &import_params) +{ + int64_t row_index = 0; + while (!buffer.is_empty()) { + const StringRef line = read_next_line(buffer); + + parse_csv_line(csv_data, row_index, line, import_params); + + row_index++; + } +} + +PointCloud *read_csv_file(const CSVImportParams &import_params) +{ + size_t buffer_len; + void *buffer = BLI_file_read_text_as_mem(import_params.filepath, 0, &buffer_len); + + if (buffer == nullptr) { + BKE_reportf(import_params.reports, + RPT_ERROR, + "CSV Import: Cannot open file '%s'", + import_params.filepath); + return nullptr; + } + + BLI_SCOPED_DEFER([&]() { MEM_freeN(buffer); }); + + StringRef buffer_str{(const char *)buffer, int64_t(buffer_len)}; + + /* Get row count and columns */ + if (buffer_str.is_empty()) { + BKE_reportf( + import_params.reports, RPT_ERROR, "CSV Import: empty file '%s'", import_params.filepath); + return nullptr; + } + + const StringRef header = read_next_line(buffer_str); + const Vector columns = get_columns(header); + + if (buffer_str.is_empty()) { + BKE_reportf(import_params.reports, + RPT_ERROR, + "CSV Import: no rows in file '%s'", + import_params.filepath); + return nullptr; + } + + /* Shallow copy buffer to preserve pointers from first row for parsing */ + StringRef data_buffer(buffer_str.begin(), buffer_str.end()); + + const StringRef first_row = read_next_line(buffer_str); + + Vector column_types; + if (!get_column_types(first_row, column_types)) { + std::string column_name = columns[column_types.size()]; + BKE_reportf(import_params.reports, + RPT_ERROR, + "CSV Import: file '%s', Column %s is of unsupported data type", + import_params.filepath, + column_name.c_str()); + return nullptr; + } + + const int64_t row_count = get_row_count(buffer_str); + + /* Create csv data */ + CsvData csv_data(row_count, columns, column_types); + + /* Fill csv data while seeking over the file */ + parse_csv_data(csv_data, data_buffer, import_params); + + return csv_data.to_point_cloud(); +} + +} // namespace blender::io::csv diff --git a/source/blender/io/csv/importer/csv_reader.hh b/source/blender/io/csv/importer/csv_reader.hh new file mode 100644 index 00000000000..1874f48b3cd --- /dev/null +++ b/source/blender/io/csv/importer/csv_reader.hh @@ -0,0 +1,18 @@ +/* SPDX-FileCopyrightText: 2024 Blender Authors + * + * SPDX-License-Identifier: GPL-2.0-or-later */ + +/** \file + * \ingroup csv + */ + +#pragma once + +struct CSVImportParams; +struct PointCloud; + +namespace blender::io::csv { + +PointCloud *read_csv_file(const CSVImportParams &import_params); + +} diff --git a/source/blender/io/wavefront_obj/CMakeLists.txt b/source/blender/io/wavefront_obj/CMakeLists.txt index eb32fd90980..61a77ac0060 100644 --- a/source/blender/io/wavefront_obj/CMakeLists.txt +++ b/source/blender/io/wavefront_obj/CMakeLists.txt @@ -28,7 +28,6 @@ set(SRC importer/obj_import_mesh.cc importer/obj_import_mtl.cc importer/obj_import_nurbs.cc - importer/obj_import_string_utils.cc importer/obj_importer.cc IO_wavefront_obj.hh @@ -44,7 +43,6 @@ set(SRC importer/obj_import_mtl.hh importer/obj_import_nurbs.hh importer/obj_import_objects.hh - importer/obj_import_string_utils.hh importer/obj_importer.hh ) @@ -67,7 +65,6 @@ blender_add_lib(bf_io_wavefront_obj "${SRC}" "${INC}" "${INC_SYS}" "${LIB}") if(WITH_GTESTS) set(TEST_SRC tests/obj_exporter_tests.cc - tests/obj_import_string_utils_tests.cc tests/obj_importer_tests.cc tests/obj_mtl_parser_tests.cc ) diff --git a/source/blender/io/wavefront_obj/importer/obj_import_file_reader.cc b/source/blender/io/wavefront_obj/importer/obj_import_file_reader.cc index 01936e41b82..1aedd38ff22 100644 --- a/source/blender/io/wavefront_obj/importer/obj_import_file_reader.cc +++ b/source/blender/io/wavefront_obj/importer/obj_import_file_reader.cc @@ -16,9 +16,10 @@ #include "BLI_string_ref.hh" #include "BLI_vector.hh" +#include "IO_string_utils.hh" + #include "obj_export_mtl.hh" #include "obj_import_file_reader.hh" -#include "obj_import_string_utils.hh" #include #include diff --git a/source/blender/makesrna/intern/rna_nodetree.cc b/source/blender/makesrna/intern/rna_nodetree.cc index a3bcc4229c4..36c8f9cc6d0 100644 --- a/source/blender/makesrna/intern/rna_nodetree.cc +++ b/source/blender/makesrna/intern/rna_nodetree.cc @@ -12479,6 +12479,7 @@ static void rna_def_nodes(BlenderRNA *brna) define("GeometryNode", "GeometryNodeImportOBJ"); define("GeometryNode", "GeometryNodeImportPLY"); define("GeometryNode", "GeometryNodeImportSTL"); + define("GeometryNode", "GeometryNodeImportCSV"); define("GeometryNode", "GeometryNodeIndexOfNearest"); define("GeometryNode", "GeometryNodeIndexSwitch", def_geo_index_switch); define("GeometryNode", "GeometryNodeInputActiveCamera"); diff --git a/source/blender/nodes/geometry/CMakeLists.txt b/source/blender/nodes/geometry/CMakeLists.txt index 7ac704c45fb..047bdb063a2 100644 --- a/source/blender/nodes/geometry/CMakeLists.txt +++ b/source/blender/nodes/geometry/CMakeLists.txt @@ -11,8 +11,9 @@ set(INC ../../makesrna ../../modifiers ../../io/common - ../../io/ply + ../../io/csv ../../io/stl + ../../io/ply ../../io/wavefront_obj # RNA_prototypes.hh ${CMAKE_BINARY_DIR}/source/blender/makesrna @@ -85,6 +86,7 @@ set(SRC nodes/node_geo_image.cc nodes/node_geo_image_info.cc nodes/node_geo_image_texture.cc + nodes/node_geo_import_csv.cc nodes/node_geo_import_obj.cc nodes/node_geo_import_ply.cc nodes/node_geo_import_stl.cc @@ -258,6 +260,14 @@ set(LIB PRIVATE bf::windowmanager ) +if(WITH_IO_CSV) + list(APPEND LIB + PRIVATE bf_io_common + PRIVATE bf_io_csv + ) + add_definitions(-DWITH_IO_CSV) +endif() + if(WITH_IO_STL) list(APPEND LIB PRIVATE bf_io_common diff --git a/source/blender/nodes/geometry/nodes/node_geo_import_csv.cc b/source/blender/nodes/geometry/nodes/node_geo_import_csv.cc new file mode 100644 index 00000000000..2c633926c00 --- /dev/null +++ b/source/blender/nodes/geometry/nodes/node_geo_import_csv.cc @@ -0,0 +1,80 @@ +/* SPDX-FileCopyrightText: 2024 Blender Authors + * + * SPDX-License-Identifier: GPL-2.0-or-later */ + +#include "BKE_report.hh" +#include "BLI_string.h" + +#include "IO_csv.hh" + +#include "node_geometry_util.hh" + +namespace blender::nodes::node_geo_import_csv { + +static void node_declare(NodeDeclarationBuilder &b) +{ + b.add_input("Path") + .subtype(PROP_FILEPATH) + .hide_label() + .description("Path to a CSV file"); + + b.add_output("Point Cloud"); +} + +static void node_geo_exec(GeoNodeExecParams params) +{ +#ifdef WITH_IO_CSV + const std::string path = params.extract_input("Path"); + if (path.empty()) { + params.set_default_remaining_outputs(); + return; + } + + blender::io::csv::CSVImportParams import_params{}; + STRNCPY(import_params.filepath, path.c_str()); + + ReportList reports; + BKE_reports_init(&reports, RPT_STORE); + BLI_SCOPED_DEFER([&]() { BKE_reports_free(&reports); }) + import_params.reports = &reports; + + PointCloud *point_cloud = blender::io::csv::import_csv_as_point_cloud(&import_params); + + LISTBASE_FOREACH (Report *, report, &(import_params.reports)->list) { + NodeWarningType type; + switch (report->type) { + case RPT_ERROR: + type = NodeWarningType::Error; + break; + default: + type = NodeWarningType::Info; + break; + } + params.error_message_add(type, TIP_(report->message)); + } + + params.set_output("Point Cloud", GeometrySet::from_pointcloud(point_cloud)); +#else + params.error_message_add(NodeWarningType::Error, + TIP_("Disabled, Blender was compiled without CSV I/O")); + params.set_default_remaining_outputs(); +#endif +} + +static void node_register() +{ + static blender::bke::bNodeType ntype; + + geo_node_type_base(&ntype, "GeometryNodeImportCSV"); + ntype.ui_name = "Import CSV"; + ntype.ui_description = "Import geometry from an CSV file"; + ntype.nclass = NODE_CLASS_INPUT; + ntype.geometry_node_execute = node_geo_exec; + ntype.declare = node_declare; + ntype.gather_link_search_ops = search_link_ops_for_import_node; + + blender::bke::node_register_type(&ntype); +} +NOD_REGISTER_NODE(node_register) + +} // namespace blender::nodes::node_geo_import_csv