diff --git a/README.md b/README.md index 8780bb9..4d3bd12 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,8 @@ C++ wrappers (using Crubit) for Rust libraries. | saphyr | [saphyr](https://crates.io/crates/saphyr) | Compiles | | serde_json | [serde_json](https://crates.io/crates/serde_json) | Compiles | | zip | [zip](https://crates.io/crates/zip) | Compiles | - +| leveldb | [rusty_leveldb](https://crates.io/crates/rusty_leveldb) | Doesn't compile - NOTE: Retry after `cc_std::virtual_unique_ptr` is supported | +| stemming | [snowball](https://crates.io/crates/snowball) | Compiles | ## Contributing diff --git a/stemming/CMakeLists.txt b/stemming/CMakeLists.txt new file mode 100644 index 0000000..d461e21 --- /dev/null +++ b/stemming/CMakeLists.txt @@ -0,0 +1,146 @@ +cmake_minimum_required(VERSION 3.22) + +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_CXX_STANDARD_REQUIRED ON) +set(CMAKE_C_COMPILER clang) +set(CMAKE_CXX_COMPILER clang++) + +project(stemming + VERSION 0.1.0 + LANGUAGES CXX C) + +include(FetchContent) + +# Fetch snowball source to get algorithms +FetchContent_Declare( + snowball_repo + GIT_REPOSITORY https://github.com/snowballstem/snowball.git + GIT_TAG main +) +FetchContent_GetProperties(snowball_repo) +if(NOT snowball_repo_POPULATED) + FetchContent_Populate(snowball_repo) +endif() + +# Rename main.rs to lib.rs to make it a library crate +if(EXISTS "${snowball_repo_SOURCE_DIR}/rust/src/main.rs") + file(RENAME "${snowball_repo_SOURCE_DIR}/rust/src/main.rs" "${snowball_repo_SOURCE_DIR}/rust/src/lib.rs") +endif() + +# 1. Build snowball_compiler +file(GLOB COMPILER_SRCS "${snowball_repo_SOURCE_DIR}/compiler/*.c") +add_executable(snowball_compiler ${COMPILER_SRCS}) +target_include_directories(snowball_compiler PRIVATE "${snowball_repo_SOURCE_DIR}/compiler") + +# 2. Find all .sbl files +file(GLOB SBL_FILES "${snowball_repo_SOURCE_DIR}/algorithms/*.sbl") + +# Extract language names +set(LANG_NAMES "") +foreach(SBL_FILE ${SBL_FILES}) + get_filename_component(LANG_NAME ${SBL_FILE} NAME_WE) + list(APPEND LANG_NAMES ${LANG_NAME}) +endforeach() + +# Sort alphabetically (matches bash sort) +list(SORT LANG_NAMES) + +# 3. Generate Rust algorithms +set(RUST_ALGORITHMS_DIR "${snowball_repo_SOURCE_DIR}/rust/src/snowball/algorithms") +file(MAKE_DIRECTORY ${RUST_ALGORITHMS_DIR}) + +set(GENERATED_RUST_ALGORITHM_FILES "") +foreach(LANG_NAME ${LANG_NAMES}) + add_custom_command( + OUTPUT "${RUST_ALGORITHMS_DIR}/${LANG_NAME}_stemmer.rs" + COMMAND snowball_compiler "${snowball_repo_SOURCE_DIR}/algorithms/${LANG_NAME}.sbl" -rust -o "${RUST_ALGORITHMS_DIR}/${LANG_NAME}_stemmer" + DEPENDS snowball_compiler "${snowball_repo_SOURCE_DIR}/algorithms/${LANG_NAME}.sbl" + COMMENT "Generating Rust code for ${LANG_NAME} stemmer" + ) + list(APPEND GENERATED_RUST_ALGORITHM_FILES "${RUST_ALGORITHMS_DIR}/${LANG_NAME}_stemmer.rs") +endforeach() + +# Custom target to drive generation +add_custom_target(generate_rust_algorithms + DEPENDS ${GENERATED_RUST_ALGORITHM_FILES} +) + +# 4. Generate generated_language_list.rs for OUR crate +set(ALGORITHM_NAMES_CONTENT "") +set(ALGORITHM_CSTR_CONTENT "") + +foreach(LANG_NAME ${LANG_NAMES}) + string(APPEND ALGORITHM_NAMES_CONTENT " \"${LANG_NAME}\",\n") + string(APPEND ALGORITHM_CSTR_CONTENT " c\"${LANG_NAME}\".as_ptr(),\n") +endforeach() + +set(GENERATED_FILE_CONTENT +"// Generated by CMake. Do not edit. +const ALGORITHM_NAMES: &[&str] = &[ +${ALGORITHM_NAMES_CONTENT}]; + +const ALGORITHM_NAMES_CSTR: &[*const std::os::raw::c_char] = &[ +${ALGORITHM_CSTR_CONTENT} std::ptr::null(), +]; +" +) + +# Write to the source directory +file(WRITE "${CMAKE_CURRENT_SOURCE_DIR}/generated_language_list.rs" "${GENERATED_FILE_CONTENT}") + +# 5. Generate Cargo.toml for OUR crate +# Copy LICENSE first +if(EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/../LICENSE") + file(COPY "${CMAKE_CURRENT_SOURCE_DIR}/../LICENSE" DESTINATION "${CMAKE_CURRENT_BINARY_DIR}") +endif() + +set(CARGO_TOML_CONTENT +"[package] +name = \"rust\" +version = \"0.1.0\" +edition = \"2021\" +license-file = \"../LICENSE\" + +[lib] +path = \"${CMAKE_CURRENT_SOURCE_DIR}/libstemmer.rs\" +crate-type = [\"staticlib\"] +doctest = false + +[dependencies] +isolang = \"2.0\" +snowball_stemmer = { package = \"testapp\", path = \"${snowball_repo_SOURCE_DIR}/rust\" } +" +) + +file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/rust/Cargo.toml" "${CARGO_TOML_CONTENT}") + +# 6. Corrosion Setup +FetchContent_Declare( + Corrosion + GIT_REPOSITORY https://github.com/thunderseethe/corrosion + GIT_TAG master +) + +FetchContent_MakeAvailable(Corrosion) + +corrosion_import_crate(MANIFEST_PATH ${CMAKE_CURRENT_BINARY_DIR}/rust/Cargo.toml) + +# Ensure algorithms are generated before building the Rust crate +add_dependencies(rust generate_rust_algorithms) +if(TARGET cargo-prebuild_rust) + add_dependencies(cargo-prebuild_rust generate_rust_algorithms) +endif() + +FetchContent_Declare( + absl + GIT_REPOSITORY https://github.com/abseil/abseil-cpp.git + FIND_PACKAGE_ARGS NAMES absl +) +FetchContent_MakeAvailable(absl) + +add_library(snowball_stemmer_wrap snowball_stemmer_wrap.cc snowball_stemmer_wrap.h) +target_link_libraries(snowball_stemmer_wrap PRIVATE rust absl::strings) +target_include_directories(snowball_stemmer_wrap PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}") + +add_executable(stemming_example main.cc) +target_link_libraries(stemming_example PRIVATE snowball_stemmer_wrap absl::strings) diff --git a/stemming/README.md b/stemming/README.md new file mode 100644 index 0000000..6511889 --- /dev/null +++ b/stemming/README.md @@ -0,0 +1,13 @@ +# Stemming + +Wrapper for the [snowball](https://crates.io/crates/snowball) crate. + +## Usage + +Requires installation of clang and a nightly version of Rust. + +``` +cd stemming +cmake -B build +cmake --build build --parallel +``` diff --git a/stemming/generated_language_list.rs b/stemming/generated_language_list.rs new file mode 100644 index 0000000..89ade80 --- /dev/null +++ b/stemming/generated_language_list.rs @@ -0,0 +1,76 @@ +const ALGORITHM_NAMES: &[&str] = &[ + "arabic", + "armenian", + "basque", + "catalan", + "czech", + "danish", + "dutch", + "dutch_porter", + "english", + "esperanto", + "estonian", + "finnish", + "french", + "german", + "greek", + "hindi", + "hungarian", + "indonesian", + "irish", + "italian", + "lithuanian", + "lovins", + "nepali", + "norwegian", + "polish", + "porter", + "portuguese", + "romanian", + "russian", + "serbian", + "spanish", + "swedish", + "tamil", + "turkish", + "yiddish", +]; + +const ALGORITHM_NAMES_CSTR: &[*const std::os::raw::c_char] = &[ + c"arabic".as_ptr(), + c"armenian".as_ptr(), + c"basque".as_ptr(), + c"catalan".as_ptr(), + c"czech".as_ptr(), + c"danish".as_ptr(), + c"dutch".as_ptr(), + c"dutch_porter".as_ptr(), + c"english".as_ptr(), + c"esperanto".as_ptr(), + c"estonian".as_ptr(), + c"finnish".as_ptr(), + c"french".as_ptr(), + c"german".as_ptr(), + c"greek".as_ptr(), + c"hindi".as_ptr(), + c"hungarian".as_ptr(), + c"indonesian".as_ptr(), + c"irish".as_ptr(), + c"italian".as_ptr(), + c"lithuanian".as_ptr(), + c"lovins".as_ptr(), + c"nepali".as_ptr(), + c"norwegian".as_ptr(), + c"polish".as_ptr(), + c"porter".as_ptr(), + c"portuguese".as_ptr(), + c"romanian".as_ptr(), + c"russian".as_ptr(), + c"serbian".as_ptr(), + c"spanish".as_ptr(), + c"swedish".as_ptr(), + c"tamil".as_ptr(), + c"turkish".as_ptr(), + c"yiddish".as_ptr(), + std::ptr::null(), +]; diff --git a/stemming/libstemmer.h b/stemming/libstemmer.h new file mode 100644 index 0000000..77be59b --- /dev/null +++ b/stemming/libstemmer.h @@ -0,0 +1,27 @@ +#ifndef SECURITY_STEMMING_LIBSTEMMER_H_ +#define SECURITY_STEMMING_LIBSTEMMER_H_ + +namespace snowball_stemmer_c { + +struct sb_stemmer; + +using sb_symbol = char; + +extern "C" { + +void sb_stemmer_delete(sb_stemmer *stemmer); + +int sb_stemmer_length(sb_stemmer *stemmer); + +const char **sb_stemmer_list(); + +sb_stemmer *sb_stemmer_new(const char *algorithm, const char *charenc); + +const sb_symbol *sb_stemmer_stem(sb_stemmer *stemmer, const sb_symbol *word, + int size); + +} // extern "C" + +} // namespace snowball_stemmer_c + +#endif // SECURITY_STEMMING_LIBSTEMMER_H_ diff --git a/stemming/libstemmer.rs b/stemming/libstemmer.rs new file mode 100644 index 0000000..244f8f1 --- /dev/null +++ b/stemming/libstemmer.rs @@ -0,0 +1,123 @@ +use snowball_stemmer::Stemmer; +use std::ffi::CStr; +use std::os::raw::c_char; +use std::os::raw::c_int; + +use isolang::Language; + +#[repr(C)] +pub struct sb_stemmer { + stemmer: Stemmer, + stem: String, + // Whenever `stem` is derived, `out_length` is equal to `stem.len()`. But in some cases (e.g. + // non-UTF8), `stem` cannot be derived, in which case `out_length` captures the length of the + // input. + out_length: i32, +} + +#[allow(non_camel_case_types)] +pub type sb_symbol = c_char; + +include!("generated_language_list.rs"); + +#[no_mangle] +pub extern "C" fn sb_stemmer_list() -> *const *const c_char { + return ALGORITHM_NAMES_CSTR.as_ptr(); +} + +#[no_mangle] +pub extern "C" fn sb_stemmer_new( + algorithm: *const c_char, + _charenc: *const c_char, +) -> *mut sb_stemmer { + assert!(!algorithm.is_null()); + let language_c_str: &CStr = unsafe { CStr::from_ptr(algorithm) }; + let language_slice: &str = language_c_str.to_str().unwrap(); + let language_name: Option<&str> = ALGORITHM_NAMES + .iter() + .position(|&name| name == language_slice) + .and_then(|i| Some(ALGORITHM_NAMES[i])) + .or_else(|| { + Language::from_639_1(language_slice) + .or_else(|| Language::from_639_3(language_slice)) + .and_then(|language| { + ALGORITHM_NAMES + .iter() + .position(|&el| el == language.to_name().to_string().to_lowercase()) + .and_then(|i| Some(ALGORITHM_NAMES[i])) + }) + }); + + (match language_name { + Some(language_name) => Box::into_raw(Box::new(sb_stemmer { + stemmer: Stemmer::create(language_name.to_string()), + stem: String::new(), + out_length: 0, + })), + None => std::ptr::null(), + }) as *mut sb_stemmer +} + +#[no_mangle] +pub extern "C" fn sb_stemmer_delete(stemmer: *mut sb_stemmer) { + if !stemmer.is_null() { + drop(unsafe { Box::from_raw(stemmer) }); + } +} + +/// Extract stem of a word. +/// +/// * `stemmer` - pointer to a stemmer. +/// * `word` - pointer to the word. +/// * `size` - bytes length of the word to stem. +/// +/// SAFETY: stemmer points to an initialized sb_stemmer, word points to a char +/// buffer, callers need to pass a size that is valid in their address space. +#[no_mangle] +pub unsafe extern "C" fn sb_stemmer_stem( + stemmer: *mut sb_stemmer, + word: *const sb_symbol, + size: c_int, +) -> *const sb_symbol { + assert!(!stemmer.is_null()); + assert!(!word.is_null()); + assert!(size >= 0); + + let stemmer = unsafe { &mut *stemmer }; + + let word_slice: &[u8] = unsafe { std::slice::from_raw_parts(word as *const u8, size as usize) }; + let word_slice: &str = match std::str::from_utf8(word_slice) { + Ok(word) => word, + Err(_) => { + stemmer.out_length = size; + return word; + } + }; + + let stem_cow = stemmer.stemmer.stem(word_slice); + match stem_cow { + std::borrow::Cow::Borrowed(b) => { + stemmer.stem.clear(); + stemmer.stem.push_str(b); + } + std::borrow::Cow::Owned(o) => { + stemmer.stem = o; + } + } + stemmer.out_length = stemmer.stem.len() as i32; + stemmer.stem.as_ptr() as *const sb_symbol +} + +/// Returns the length of the stem of the last word. +/// +/// * `stemmer` - pointer to a stemmer. +/// +/// SAFETY: stemmer points to an initialized sb_stemmer. +#[no_mangle] +pub unsafe extern "C" fn sb_stemmer_length(stemmer: *mut sb_stemmer) -> c_int { + if stemmer.is_null() { + return -1; + } + + return unsafe { (*stemmer).out_length }; +} diff --git a/stemming/main.cc b/stemming/main.cc new file mode 100644 index 0000000..4229de1 --- /dev/null +++ b/stemming/main.cc @@ -0,0 +1,86 @@ +#include +#include +#include + +#include "snowball_stemmer_wrap.h" + +void RunDemonstration() { + struct StemExample { + const char* lang; + const char* word; + }; + + const StemExample examples[] = { + {"en", "broadening"}, + {"en", "abattement"}, + {"fr", "majestueuse"}, + {"de", "kätzchen"}, + {"de", "halstücher"}, + {"es", "torpedearon"}, + {"nl", "lichtgevoeligheid"}, + {"it", "pronuncerà"}, + {"sv", "kloekornas"}, + {"da", "undertrykkerens"}, + {"fi", "innostuessaan"}, + {"pt", "quimioterápicos"}, + {"ru", "валяется"}, + {"id", "berpemandangan"}, + {"ta", "இக்கதையின்"}, + {"lt", "katėmis"}, + }; + + std::cout << "=== Snowball Stemmer Demonstration ===\n"; + for (const auto& ex : examples) { + security::stemmer::SnowballStemmer stemmer(ex.lang); + if (!stemmer.IsSupportedLanguage()) { + std::cerr << "Language '" << ex.lang << "' is not supported.\n"; + continue; + } + + std::string output; + if (stemmer.StemUTF8(ex.word, &output)) { + std::cout << "[" << ex.lang << "] '" << ex.word << "' -> '" << output + << "'\n"; + } else { + std::cerr << "Failed to stem '" << ex.word << "' in " << ex.lang + << "\n"; + } + } + std::cout << "=====================================\n"; +} + +int main(int argc, char** argv) { + if (argc <= 1) { + RunDemonstration(); + std::cout << "Usage: " << argv[0] << " \n"; + std::cout << "Example: " << argv[0] << " en powered\n"; + return 0; + } + + if (argc < 3) { + std::cerr << "Error: Missing word to stem.\n"; + std::cerr << "Usage: " << argv[0] << " \n"; + std::cerr << "Example: " << argv[0] << " en powered\n"; + return 1; + } + + const char* const lang = argv[1]; + + security::stemmer::SnowballStemmer stemmer(lang); + if (!stemmer.IsSupportedLanguage()) { + std::cerr << "Language '" << lang << "' is not supported.\n"; + return 1; + } + + const char* const input = argv[2]; + + std::string output; + if (stemmer.StemUTF8(input, &output)) { + std::cout << "Stem of '" << input << "' in " << lang << " is '" << output + << "'\n"; + } else { + std::cerr << "Failed to stem '" << input << "'\n"; + } + + return 0; +} diff --git a/stemming/rust/Cargo.toml b/stemming/rust/Cargo.toml new file mode 100644 index 0000000..9d7bc28 --- /dev/null +++ b/stemming/rust/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "rust" +version = "0.1.0" +edition = "2021" +license-file = "../../LICENSE" + +[lib] +path = "../libstemmer.rs" +crate-type = ["staticlib"] +doctest = false + +[dependencies] +isolang = "2.0" +snowball_stemmer = { package = "rust-stemmers", version = "1.2" } diff --git a/stemming/snowball_stemmer_wrap.cc b/stemming/snowball_stemmer_wrap.cc new file mode 100644 index 0000000..4cb0252 --- /dev/null +++ b/stemming/snowball_stemmer_wrap.cc @@ -0,0 +1,51 @@ +// Copied form google3/third_party/porter_stemmer_2013/porter-stemmer-wrap.cc +// Wrap Snowball stemmer for Google use. + +#include "snowball_stemmer_wrap.h" + +#include +#include + +#include "libstemmer.h" +#include "absl/strings/string_view.h" + +using snowball_stemmer_c::sb_stemmer_delete; +using snowball_stemmer_c::sb_stemmer_length; +using snowball_stemmer_c::sb_stemmer_new; +using snowball_stemmer_c::sb_stemmer_stem; +using snowball_stemmer_c::sb_symbol; + +namespace security::stemmer { + +SnowballStemmer::SnowballStemmer(const char* lang_code) + : stemmer_(sb_stemmer_new(lang_code, nullptr /* UTF-8 */)), + lang_code_(lang_code) {} + +SnowballStemmer::~SnowballStemmer() { sb_stemmer_delete(stemmer_); } + +bool SnowballStemmer::IsSupportedLanguage() const { + return stemmer_ != nullptr; +} + +bool SnowballStemmer::StemUTF8(const absl::string_view &input, + std::string *output) { + if (stemmer_ == nullptr || input.size() > kMaxStemTermLength) { + output->assign(input.data(), input.size()); + return false; + } + + const sb_symbol *stem = sb_stemmer_stem( + stemmer_, reinterpret_cast(input.data()), + static_cast(input.size())); + + if (stem == nullptr) { + output->assign(input.data(), input.size()); + return false; + } + + output->assign(reinterpret_cast(stem), + static_cast(sb_stemmer_length(stemmer_))); + return true; +} + +} // namespace security::stemmer diff --git a/stemming/snowball_stemmer_wrap.h b/stemming/snowball_stemmer_wrap.h new file mode 100644 index 0000000..6dde474 --- /dev/null +++ b/stemming/snowball_stemmer_wrap.h @@ -0,0 +1,62 @@ +// Copied from google3/third_party/porter_stemmer_2013/porter-stemmer-wrap.h +// +// Wraps a snowball stemmer for a given language. +// If the language is not supported, IsSupportedLanguage returns false and +// the StemUTF8 method is a pass-through. +// +// Porter stemmers are re-entrant but not thread safe, thus it is not safe +// to share PorterStemmer instances between threads. + +#ifndef SECURITY_STEMMING_SNOWBALL_STEMMER_WRAP_H_ +#define SECURITY_STEMMING_SNOWBALL_STEMMER_WRAP_H_ + +#include +#include + +#include "absl/strings/string_view.h" + +namespace snowball_stemmer_c { +struct sb_stemmer; +} // namespace snowball_stemmer_c + +namespace security::stemmer { + +// Input larger than this is skipped. +const size_t kMaxStemTermLength = 512; + +class SnowballStemmer { + public: + // Instantiates a Snowball stemmer for the given language code. + // For a list of supported languages, please refer to: + // https://source.corp.google.com/piper///depot/google3/security/stemming/libstemmer.rs;l=18-44 + // A Snowball stemmer object is re-entrant but is not thread safe. + explicit SnowballStemmer(const char *lang_code); + + // This type is neither copyable nor movable. + SnowballStemmer(const SnowballStemmer &) = delete; + SnowballStemmer &operator=(const SnowballStemmer &) = delete; + + ~SnowballStemmer(); + + // Returns true if the stemmer was correctly initialized and the language + // is supported by an underlying Porter stemmer. + // Returns false otherwise. + bool IsSupportedLanguage() const; + + // Returns true if the underlying stemmer successfully processed the input + // word, false otherwise. + // In either case, the input word to be stemmed is copied to the output + // string allowing for silent pass-through operation. + bool StemUTF8(const absl::string_view &input, std::string *output); + + // Returns the lang_code passed during construction. + inline const std::string &lang_code() const { return lang_code_; } + + private: + struct snowball_stemmer_c::sb_stemmer *stemmer_; + const std::string lang_code_; +}; + +} // namespace security::stemmer + +#endif // SECURITY_STEMMING_SNOWBALL_STEMMER_WRAP_H_