This TaskGenerator enables the calculation of evaluation metrics on top-k style sparse predictions. More...

#include <evaluate.h>

Inheritance diagram for dismec::prediction::EvaluateMetrics:

Public Types
using	LabelList = std::vector< std::vector< label_id_t > >

using	prediction_t = Eigen::Ref< const Eigen::Matrix< long, 1, Eigen::Dynamic > >

Public Types inherited from dismec::parallel::TaskGenerator
using	thread_id_t = dismec::parallel::thread_id_t

Public Member Functions
	EvaluateMetrics (const LabelList sparse_labels, const IndexMatrix sparse_predictions, long num_labels)

	~EvaluateMetrics () override

void	prepare (long num_threads, long chunk_size) override
	Called to notify the `TaskGenerator` about the number of threads. More...

void	init_thread (thread_id_t thread_id) override
	Called once a thread has spun up, but before it runs its first task. More...

void	add_precision_at_k (long k, std::string name={})

void	add_dcg_at_k (long k, bool normalize, std::string name={})

void	add_abandonment_at_k (long k, std::string name={})

MacroMetricReporter *	add_macro_at_k (long k)

std::vector< std::pair< std::string, double > >	get_metrics () const

void	run_task (long task_id, thread_id_t thread_id)

void	run_tasks (long begin, long end, thread_id_t thread_id) override

void	finalize () override
	Called after all threads have finished their tasks. More...

long	num_tasks () const override

Public Member Functions inherited from dismec::parallel::TaskGenerator
virtual	~TaskGenerator ()=default

Static Public Member Functions
static void	process_prediction (const std::vector< label_id_t > &raw_labels, const prediction_t &raw_prediction, std::vector< sTrueLabelInfo > &proc_labels, std::vector< sPredLabelInfo > &proc_pred)

Private Attributes
const LabelList *	m_Labels

const IndexMatrix *	m_Predictions

long	m_NumLabels

std::vector< std::vector< std::unique_ptr< MetricCollectionInterface > > >	m_Collectors

std::vector< std::unique_ptr< MetricReportInterface > >	m_Metrics

std::vector< std::vector< sTrueLabelInfo > >	m_ThreadLocalTrueLabels

std::vector< std::vector< sPredLabelInfo > >	m_ThreadLocalPredictedLabels

Detailed Description

This TaskGenerator enables the calculation of evaluation metrics on top-k style sparse predictions.

In order to enable efficient calculation of multiple metrics form a given instance of sparse predictions, this class generates per-thread caches that determine which labels are true/false positives and false negatives.

Definition at line 34 of file evaluate.h.

Member Typedef Documentation

◆ LabelList

using dismec::prediction::EvaluateMetrics::LabelList = std::vector<std::vector<label_id_t> >

Definition at line 36 of file evaluate.h.

◆ prediction_t

using dismec::prediction::EvaluateMetrics::prediction_t = Eigen::Ref<const Eigen::Matrix<long, 1, Eigen::Dynamic> >

Definition at line 58 of file evaluate.h.

Constructor & Destructor Documentation

◆ EvaluateMetrics()

EvaluateMetrics::EvaluateMetrics	(	const LabelList *	sparse_labels,
		const IndexMatrix *	sparse_predictions,
		long	num_labels
	)

Definition at line 13 of file evaluate.cpp.

References m_Collectors, m_Labels, m_Predictions, and dismec::ssize().

◆ ~EvaluateMetrics()

EvaluateMetrics::~EvaluateMetrics ( )

overridedefault

Member Function Documentation

◆ add_abandonment_at_k()

void EvaluateMetrics::add_abandonment_at_k	(	long	k,
		std::string	name = `{}`
	)

Definition at line 106 of file evaluate.cpp.

References m_Collectors, m_Metrics, m_NumLabels, and m_Predictions.

◆ add_dcg_at_k()

void EvaluateMetrics::add_dcg_at_k	(	long	k,
		bool	normalize,
		std::string	name = `{}`
	)

Definition at line 85 of file evaluate.cpp.

References m_Collectors, m_Metrics, m_NumLabels, and m_Predictions.

◆ add_macro_at_k()

MacroMetricReporter * EvaluateMetrics::add_macro_at_k ( long k )

Definition at line 120 of file evaluate.cpp.

References m_Collectors, m_Metrics, m_NumLabels, and m_Predictions.

◆ add_precision_at_k()

void EvaluateMetrics::add_precision_at_k	(	long	k,
		std::string	name = `{}`
	)

Definition at line 71 of file evaluate.cpp.

References m_Collectors, m_Metrics, m_NumLabels, and m_Predictions.

◆ finalize()

void EvaluateMetrics::finalize ( )

overridevirtual

Called after all threads have finished their tasks.

This function is called from the main thread after all worker threads have finished their work. It can be used to perform single threaded reductions or clean up per-thread buffers.

Reimplemented from dismec::parallel::TaskGenerator.

Definition at line 149 of file evaluate.cpp.

References m_Collectors, and dismec::ssize().

◆ get_metrics()

std::vector< std::pair< std::string, double > > EvaluateMetrics::get_metrics ( ) const

Definition at line 134 of file evaluate.cpp.

References m_Metrics.

◆ init_thread()

void EvaluateMetrics::init_thread ( parallel::thread_id_t thread_id )

overridevirtual

Called once a thread has spun up, but before it runs its first task.

This function is called from inside the thread that also will run the tasks.

Reimplemented from dismec::parallel::TaskGenerator.

Definition at line 157 of file evaluate.cpp.

References m_Collectors, and dismec::opaque_int_type< Tag, T >::to_index().

◆ num_tasks()

long EvaluateMetrics::num_tasks ( ) const

overridevirtual

Implements dismec::parallel::TaskGenerator.

Definition at line 67 of file evaluate.cpp.

References m_Predictions.

◆ prepare()

void EvaluateMetrics::prepare	(	long	num_threads,
		long	chunk_size
	)

overridevirtual

Called to notify the TaskGenerator about the number of threads.

This function is called from the main thread, before distributed work is started. It gives the TaskGenerator a chance to allocate working memory for each thread, so these allocations don't need to be done and repeated in run_task().

Note: For memory that is used inside the computations done by each thread, the init_thread() should be used. This will be called from inside the thread that will do the actual computations, so that when using this on a NUMA system, first-touch policy has a chance to place the allocation in the correct RAM. In that case, this function should only allocate an array of pointers that will be filled in by init_thread().

Parameters

num_threads	Number of threads that will be used.
chunk_size	A hint for the size of chunks used when running this task. Note that if the total number of tasks is not a multiple of the `chunk_size`, there may be some calls to `run_tasks()` with less than `chunk_size` tasks.

Reimplemented from dismec::parallel::TaskGenerator.

Definition at line 143 of file evaluate.cpp.

References m_Collectors, m_ThreadLocalPredictedLabels, and m_ThreadLocalTrueLabels.

◆ process_prediction()

void EvaluateMetrics::process_prediction	(	const std::vector< label_id_t > &	raw_labels,
		const prediction_t &	raw_prediction,
		std::vector< sTrueLabelInfo > &	proc_labels,
		std::vector< sPredLabelInfo > &	proc_pred
	)

static

Definition at line 24 of file evaluate.cpp.

Referenced by run_task(), and anonymous_namespace{metrics.cpp}::update_metric().

◆ run_task()

void EvaluateMetrics::run_task	(	long	task_id,
		thread_id_t	thread_id
	)

Definition at line 47 of file evaluate.cpp.

References m_Collectors, m_Predictions, m_ThreadLocalPredictedLabels, m_ThreadLocalTrueLabels, process_prediction(), and dismec::opaque_int_type< Tag, T >::to_index().

Referenced by run_tasks().

◆ run_tasks()

void EvaluateMetrics::run_tasks	(	long	begin,
		long	end,
		thread_id_t	thread_id
	)

overridevirtual

Implements dismec::parallel::TaskGenerator.

Definition at line 61 of file evaluate.cpp.

References run_task().

Member Data Documentation

◆ m_Collectors

std::vector<std::vector<std::unique_ptr<MetricCollectionInterface> > > dismec::prediction::EvaluateMetrics::m_Collectors

private

Definition at line 67 of file evaluate.h.

Referenced by add_abandonment_at_k(), add_dcg_at_k(), add_macro_at_k(), add_precision_at_k(), EvaluateMetrics(), finalize(), init_thread(), prepare(), and run_task().

◆ m_Labels

const LabelList* dismec::prediction::EvaluateMetrics::m_Labels

private

Definition at line 63 of file evaluate.h.

Referenced by EvaluateMetrics().

◆ m_Metrics

std::vector<std::unique_ptr<MetricReportInterface> > dismec::prediction::EvaluateMetrics::m_Metrics

private

Definition at line 68 of file evaluate.h.

Referenced by add_abandonment_at_k(), add_dcg_at_k(), add_macro_at_k(), add_precision_at_k(), and get_metrics().

◆ m_NumLabels

long dismec::prediction::EvaluateMetrics::m_NumLabels

private

Definition at line 65 of file evaluate.h.

Referenced by add_abandonment_at_k(), add_dcg_at_k(), add_macro_at_k(), and add_precision_at_k().

◆ m_Predictions

const IndexMatrix* dismec::prediction::EvaluateMetrics::m_Predictions

private

Definition at line 64 of file evaluate.h.

Referenced by add_abandonment_at_k(), add_dcg_at_k(), add_macro_at_k(), add_precision_at_k(), EvaluateMetrics(), num_tasks(), and run_task().

◆ m_ThreadLocalPredictedLabels

std::vector<std::vector<sPredLabelInfo> > dismec::prediction::EvaluateMetrics::m_ThreadLocalPredictedLabels

private

Definition at line 72 of file evaluate.h.

Referenced by prepare(), and run_task().

◆ m_ThreadLocalTrueLabels

std::vector<std::vector<sTrueLabelInfo> > dismec::prediction::EvaluateMetrics::m_ThreadLocalTrueLabels

private

Definition at line 71 of file evaluate.h.

Referenced by prepare(), and run_task().

The documentation for this class was generated from the following files:

src/prediction/evaluate.h
src/prediction/evaluate.cpp

Public Types

Public Member Functions

Static Public Member Functions

Private Attributes

Detailed Description

Member Typedef Documentation

◆ LabelList

◆ prediction_t

Constructor & Destructor Documentation

◆ EvaluateMetrics()

◆ ~EvaluateMetrics()

Member Function Documentation

◆ add_abandonment_at_k()

◆ add_dcg_at_k()

◆ add_macro_at_k()

◆ add_precision_at_k()

◆ finalize()

◆ get_metrics()

◆ init_thread()

◆ num_tasks()

◆ prepare()

◆ process_prediction()

◆ run_task()

◆ run_tasks()

Member Data Documentation

◆ m_Collectors

◆ m_Labels

◆ m_Metrics

◆ m_NumLabels

◆ m_Predictions

◆ m_ThreadLocalPredictedLabels

◆ m_ThreadLocalTrueLabels