Commit e3747a2f authored by Jan Lukas Steimann's avatar Jan Lukas Steimann
Browse files

Add first version for dataset chapter

parent d702592f
\section{Dataset}
\subsection{Dataset}
\subsection{Corpus}
\begin{frame}
This is the second slide \cite{wachsmuth:2017a}.
\frametitle{Corpus}
\begin{itemize}
\item For our study, we used the Webis-ArgRank2017 dataset from Wachsmuth et
al.\footnote[1]{H. Wachsmuth, B. Stein, and Y. Ajjour "PageRank" for Argument
Relevance}
\item In this dataset Wachsmuth et al. constructed a ground-truth argument graph
as well as benchmark for argument ranking from this argument graph
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Corpus}
\begin{itemize}
\item The data are originally collected from the Argument Web and stored in an
argument graph
\item The Argument Web was the largest existing argument database with a
structured argument corpora at that time
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Corpus}
\begin{itemize}
\item In the resulting argument graph G = (A, E)
\begin{itemize}
\item Each node represents $a_i \in A$ an argument consisting of a conclusion
$c_i$ and a not-empty set of premises $P_i$ $\Rightarrow$ $a_i = \langle c_i, P_i \rangle$
\item An edge $(a_j, a_i)$ is given if the conclusion $a_j$ is used as a premise
of $a_i$
\item Consequently, $P_i = \{c_1,...,c_k\}, k \geq 1$
\end{itemize}
\end{itemize}
\begin{figure}
\includegraphics[width=0.4\linewidth]{bilder/DatasetLocalView2.png}
\caption{Argument Graph from Argument Web}
\end{figure}
\end{frame}
\subsection{Benchmark Argument Ranking}
\begin{frame}
\frametitle{Benchmark Argument Ranking}
\begin{itemize}
\item To create the benchmark dataset, Wachsmuth et al. only kept arguments
from the graph that fulfill their requirements
\begin{itemize}
\item If a conclusion was part in more than one argument, it was kept
\item Furhtermore, Wachsmuth et al. removed all nodes that do not contain a
real claim
\item Additionally, an argument:
\begin{itemize}
\item has to be a valid-counter argument
\item must be based on reasonalb premises
\item must allow a logic interference to be drawn
\end{itemize}
\end{itemize}
\end{itemize}
\end{frame}
\begin{frame}
\frametitle{Benchmark Argument Ranking}
\begin{itemize}
\item The resulting benchmark dataset consist of 32 conclusions which
participated in 110 arguments
\item These 110 arguements were ranked by seven experts from computational
linguistics and information retireval
\item Each argument was ranked by how much each of its premises contributes to
the acceptance or rejection of the conlusion
\end{itemize}
\end{frame}
\subsection{Evaluation Method}
\begin{frame}
\frametitle{Evaluation Method}
\begin{itemize}
\item To evaluate the agreement between the experts and to ensure comparability
were than used Kendall's $\tau$
\item Kendall $\tau$ is correlation coefficient that indicates the agreement
between two quantities with respect to a property
\begin{itemize}
\item In this case, this means the agreement between two experts with respect to
an argument
\item -1 signifies a complete disagreement and +1 a complete agreement
\end{itemize}
\item The mean over all experts for the evaluation of the benchmark is 0.36
\end{itemize}
\end{frame}
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment