Struct SstableStore

pub struct SstableStore {
    path: String,
    store: ObjectStoreRef,
    meta_cache: HybridCache<HummockSstableObjectId, Box<Sstable>>,
    block_cache: HybridCache<SstableBlockIndex, Box<Block>>,
    recent_filter: Option<Arc<RecentFilter<(HummockSstableObjectId, usize)>>>,
    prefetch_buffer_usage: Arc<AtomicUsize>,
    prefetch_buffer_capacity: usize,
    max_prefetch_block_number: usize,
    use_new_object_prefix_strategy: bool,
}

Fields

path: String

store: ObjectStoreRef

meta_cache: HybridCache<HummockSstableObjectId, Box<Sstable>>

block_cache: HybridCache<SstableBlockIndex, Box<Block>>

recent_filter: Option<Arc<RecentFilter<(HummockSstableObjectId, usize)>>>

Recent filter for (sst_obj_id, blk_idx).

blk_idx == USIZE::MAX stands for sst_obj_id only entry.

prefetch_buffer_usage: Arc<AtomicUsize>

prefetch_buffer_capacity: usize

max_prefetch_block_number: usize

use_new_object_prefix_strategy: bool

Whether the object store is divided into prefixes depends on two factors:

1. The specific object store type.
2. Whether the existing cluster is a new cluster.

The value of use_new_object_prefix_strategy is determined by the use_new_object_prefix_strategy field in the system parameters. For a new cluster, use_new_object_prefix_strategy is set to True. For an old cluster, use_new_object_prefix_strategy is set to False. The final decision of whether to divide prefixes is based on this field and the specific object store type, this approach is implemented to ensure backward compatibility.

Implementations

impl SstableStore

pub fn new(config: SstableStoreConfig) -> Self

pub async fn for_compactor( store: ObjectStoreRef, path: String, block_cache_capacity: usize, meta_cache_capacity: usize, use_new_object_prefix_strategy: bool, ) -> HummockResult<Self>

For compactor, we do not need a high concurrency load for cache. Instead, we need the cache can be evict more effective.

pub async fn delete( &self, object_id: HummockSstableObjectId, ) -> HummockResult<()>

pub fn delete_cache( &self, object_id: HummockSstableObjectId, ) -> HummockResult<()>

pub(crate) async fn put_sst_data( &self, object_id: HummockSstableObjectId, data: Bytes, ) -> HummockResult<()>

pub async fn prefetch_blocks( &self, sst: &Sstable, block_index: usize, end_index: usize, policy: CachePolicy, stats: &mut StoreLocalStatistic, ) -> HummockResult<Box<dyn BlockStream>>

pub async fn get_block_response( &self, sst: &Sstable, block_index: usize, policy: CachePolicy, stats: &mut StoreLocalStatistic, ) -> HummockResult<BlockResponse>

pub async fn get( &self, sst: &Sstable, block_index: usize, policy: CachePolicy, stats: &mut StoreLocalStatistic, ) -> HummockResult<BlockHolder>

pub fn get_sst_data_path(&self, object_id: HummockSstableObjectId) -> String

pub fn get_object_id_from_path(path: &str) -> HummockSstableObjectId

pub fn store(&self) -> ObjectStoreRef

pub async fn clear_block_cache(&self) -> HummockResult<()>

pub async fn clear_meta_cache(&self) -> HummockResult<()>

pub async fn sstable_cached( &self, sst_obj_id: HummockSstableObjectId, ) -> HummockResult<Option<HybridCacheEntry<HummockSstableObjectId, Box<Sstable>>>>

pub fn sstable( &self, sst: &SstableInfo, stats: &mut StoreLocalStatistic, ) -> impl Future<Output = HummockResult<TableHolder>> + Send + 'static

Returns table_holder

pub async fn list_object_metadata_from_object_store( &self, prefix: Option<String>, start_after: Option<String>, limit: Option<usize>, ) -> HummockResult<ObjectMetadataIter>

pub fn create_sst_writer( self: Arc<Self>, object_id: HummockSstableObjectId, options: SstableWriterOptions, ) -> BatchUploadWriter

pub fn insert_meta_cache( &self, object_id: HummockSstableObjectId, meta: SstableMeta, )

pub fn insert_block_cache( &self, object_id: HummockSstableObjectId, block_index: u64, block: Box<Block>, )

pub fn get_meta_memory_usage(&self) -> u64

pub fn get_prefetch_memory_usage(&self) -> usize

pub async fn get_stream_for_blocks( &self, object_id: HummockSstableObjectId, metas: &[BlockMeta], ) -> HummockResult<BlockDataStream>

pub fn data_recent_filter( &self, ) -> Option<&Arc<RecentFilter<(HummockSstableObjectId, usize)>>>

pub fn meta_cache(&self) -> &HybridCache<HummockSstableObjectId, Box<Sstable>>

pub fn block_cache(&self) -> &HybridCache<SstableBlockIndex, Box<Block>>

pub fn recent_filter( &self, ) -> Option<&Arc<RecentFilter<(HummockSstableObjectId, usize)>>>

pub async fn create_streaming_uploader( &self, path: &str, ) -> ObjectResult<ObjectStreamingUploader>