TiKV panic: range end index 182 out of range for slice of length 180 in table_scan_executor

[ekexium]

Bug Report

What version of TiKV are you using?

master 86e0ec3006dd306a254481a3631069370b3a0537

What operating system and CPU are you using?

Steps to reproduce

No idea how it happened. I was modifying memdb in client-go. I ran large insert/update/delete statements multiple times.
Some time later I noticed TiKV keeps restarting, and I found logs:

[lib.rs:477] ["range end index 182 out of range for slice of length 180"] [backtrace="   0: tikv_util::set_panic_hook::{{closure}}
             at /workspace/source/tikv/components/tikv_util/src/lib.rs:476:18
   1: <alloc::boxed::Box<F,A> as core::ops::function::Fn<Args>>::call
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2029:9
      std::panicking::rust_panic_with_hook
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:783:13
   2: std::panicking::begin_panic_handler::{{closure}}
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:657:13
   3: std::sys_common::backtrace::__rust_end_short_backtrace
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:171:18
   4: rust_begin_unwind
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:645:5
   5: core::panicking::panic_fmt
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panicking.rs:72:14
   6: core::slice::index::slice_end_index_len_fail_rt
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/slice/index.rs:76:5
      core::slice::index::slice_end_index_len_fail
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/slice/index.rs:68:9
   7: <core::ops::range::Range<usize> as core::slice::index::SliceIndex<[T]>>::index
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/slice/index.rs:394:13
      core::slice::index::<impl core::ops::index::Index<I> for [T]>::index
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/slice/index.rs:18:9
      tidb_query_executors::table_scan_executor::TableScanExecutorImpl::process_v2
             at /workspace/source/tikv/components/tidb_query_executors/src/table_scan_executor.rs:249:53
      <tidb_query_executors::table_scan_executor::TableScanExecutorImpl as tidb_query_executors::util::scan_executor::ScanExecutorImpl>::process_kv_pair
             at /workspace/source/tikv/components/tidb_query_executors/src/table_scan_executor.rs:356:43
   8: tidb_query_executors::util::scan_executor::ScanExecutor<S,I,F>::fill_column_vec::{{closure}}
             at /workspace/source/tikv/components/tidb_query_executors/src/util/scan_executor.rs:116:33
      <tidb_query_executors::util::scan_executor::ScanExecutor<S,I,F> as tidb_query_executors::interface::BatchExecutor>::next_batch::{{closure}}
             at /workspace/source/tikv/components/tidb_query_executors/src/util/scan_executor.rs:179:80
   9: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/future.rs:124:9
      <tidb_query_executors::table_scan_executor::BatchTableScanExecutor<S,F> as tidb_query_executors::interface::BatchExecutor>::next_batch::{{closure}}
             at /workspace/source/tikv/components/tidb_query_executors/src/table_scan_executor.rs:126:38
  10: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/future.rs:124:9
      <tikv_util::quota_limiter::CpuObserveFuture<F> as core::future::future::Future>::poll
             at /workspace/source/tikv/components/tikv_util/src/quota_limiter.rs:177:19
  11: tikv::coprocessor::statistics::analyze::RowSampleBuilder<S,F>::collect_column_stats::{{closure}}
             at /workspace/source/tikv/src/coprocessor/statistics/analyze.rs:108:26
  12: tikv::coprocessor::statistics::analyze_context::AnalyzeContext<S,F>::handle_full_sampling::{{closure}}
             at /workspace/source/tikv/src/coprocessor/statistics/analyze_context.rs:123:57
      <tikv::coprocessor::statistics::analyze_context::AnalyzeContext<S,F> as tikv::coprocessor::RequestHandler>::handle_request::{{closure}}
             at /workspace/source/tikv/src/coprocessor/statistics/analyze_context.rs:285:78
  13: <core::pin::Pin<P> as core::future::future::Future>::poll
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/future/future.rs:124:9
      <tikv::coprocessor::interceptors::deadline::DeadlineChecker<F> as core::future::future::Future>::poll
             at /workspace/source/tikv/src/coprocessor/interceptors/deadline.rs:37:9
      <tikv::coprocessor::interceptors::tracker::Tracker<F,E> as core::future::future::Future>::poll
             at /workspace/source/tikv/src/coprocessor/interceptors/tracker.rs:54:19
  14: <tikv::coprocessor::interceptors::concurrency_limiter::ConcurrencyLimiter<PF,F> as core::future::future::Future>::poll
             at /workspace/source/tikv/src/coprocessor/interceptors/concurrency_limiter.rs:112:15
      tikv::coprocessor::endpoint::Endpoint<E>::handle_unary_request_impl::{{closure}}
             at /workspace/source/tikv/src/coprocessor/endpoint.rs:463:87
      <resource_metering::InTags<T> as core::future::future::Future>::poll
             at /workspace/source/tikv/components/resource_metering/src/lib.rs:267:9
      <futures_util::future::future::map::Map<Fut,F> as core::future::future::Future>::poll
             at /workspace/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.15/src/future/future/map.rs:55:37
      <futures_util::future::future::Map<Fut,F> as core::future::future::Future>::poll
             at /workspace/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.15/src/lib.rs:93:13
  15: <futures_util::future::future::map::Map<Fut,F> as core::future::future::Future>::poll
             at /workspace/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.15/src/future/future/map.rs:55:37
      <futures_util::future::future::Map<Fut,F> as core::future::future::Future>::poll
             at /workspace/.cargo/registry/src/index.crates.io-6f17d22bba15001f/futures-util-0.3.15/src/lib.rs:93:13
      <resource_control::future::ControlledFuture<F> as core::future::future::Future>::poll
             at /workspace/source/tikv/components/resource_control/src/future.rs:48:19
  16: resource_control::future::with_resource_limiter::{{closure}}
             at /workspace/source/tikv/components/resource_control/src/future.rs:240:11
      <tracker::tls::TrackedFuture<F> as core::future::future::Future>::poll::{{closure}}
             at /workspace/source/tikv/components/tracker/src/tls.rs:64:23
      std::thread::local::LocalKey<T>::try_with
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:270:16
      std::thread::local::LocalKey<T>::with
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/local.rs:246:9
      <tracker::tls::TrackedFuture<F> as core::future::future::Future>::poll
             at /workspace/source/tikv/components/tracker/src/tls.rs:62:27
      yatp::task::future::RawTask<F>::poll
             at /workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/task/future.rs:59:9
  17: yatp::task::future::TaskCell::poll
             at /workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/task/future.rs:103:9
      <yatp::task::future::Runner as yatp::pool::runner::Runner>::handle
             at /workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/task/future.rs:387:20
  18: <tikv_util::yatp_pool::YatpPoolRunner<T> as yatp::pool::runner::Runner>::handle
             at /workspace/source/tikv/components/tikv_util/src/yatp_pool/mod.rs:199:24
      <yatp::queue::multilevel::TrackedRunner<R> as yatp::pool::runner::Runner>::handle
             at /workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/queue/multilevel.rs:285:19
  19: yatp::pool::worker::WorkerThread<T,R>::run
             at /workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/pool/worker.rs:48:13
      yatp::pool::builder::LazyBuilder<T>::build::{{closure}}
             at /workspace/.cargo/git/checkouts/yatp-e704b73c3ee279b6/793be4d/src/pool/builder.rs:114:25
      std::sys_common::backtrace::__rust_begin_short_backtrace
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys_common/backtrace.rs:155:18
  20: std::thread::Builder::spawn_unchecked_::{{closure}}::{{closure}}
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:529:17
      <core::panic::unwind_safe::AssertUnwindSafe<F> as core::ops::function::FnOnce<()>>::call_once
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/panic/unwind_safe.rs:272:9
      std::panicking::try::do_call
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:552:40
      std::panicking::try
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panicking.rs:516:19
      std::panic::catch_unwind
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/panic.rs:142:14
      std::thread::Builder::spawn_unchecked_::{{closure}}
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/thread/mod.rs:528:30
      core::ops::function::FnOnce::call_once{{vtable.shim}}
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/core/src/ops/function.rs:250:5
  21: <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9
      <alloc::boxed::Box<F,A> as core::ops::function::FnOnce<Args>>::call_once
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/alloc/src/boxed.rs:2015:9
      std::sys::unix::thread::Thread::new::thread_start
             at /root/.rustup/toolchains/nightly-2023-12-28-x86_64-unknown-linux-gnu/lib/rustlib/src/rust/library/std/src/sys/unix/thread.rs:108:17
  22: start_thread
  23: __GI___clone3
"] [location=components/tidb_query_executors/src/table_scan_executor.rs:249] [thread_name=unified-read-pool-5] [thread_id=38]

What did you expect?

No panicking.

What did happened?

TiKV keeps restarting.