diff --git a/slides/dataset.tex b/slides/dataset.tex index 8839aa7492fd551f3aff71c7d1d438b76b55a89a..38838102e44afdcc38327cea579702b982daf5f5 100644 --- a/slides/dataset.tex +++ b/slides/dataset.tex @@ -1,5 +1,93 @@ \section{Dataset} -\subsection{Dataset} + +\subsection{Corpus} \begin{frame} - This is the second slide \cite{wachsmuth:2017a}. + \frametitle{Corpus} + \begin{itemize} + \item For our study, we used the Webis-ArgRank2017 dataset from Wachsmuth et + al.\footnote[1]{H. Wachsmuth, B. Stein, and Y. Ajjour "PageRank" for Argument + Relevance} + \item In this dataset Wachsmuth et al. constructed a ground-truth argument graph + as well as benchmark for argument ranking from this argument graph + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Corpus} + \begin{itemize} + \item The data are originally collected from the Argument Web and stored in an + argument graph + \item The Argument Web was the largest existing argument database with a + structured argument corpora at that time + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Corpus} + \begin{itemize} + \item In the resulting argument graph G = (A, E) + \begin{itemize} + \item Each node represents $a_i \in A$ an argument consisting of a conclusion + $c_i$ and a not-empty set of premises $P_i$ $\Rightarrow$ $a_i = \langle c_i, P_i \rangle$ + \item An edge $(a_j, a_i)$ is given if the conclusion $a_j$ is used as a premise + of $a_i$ + \item Consequently, $P_i = \{c_1,...,c_k\}, k \geq 1$ + \end{itemize} + \end{itemize} + + \begin{figure} + \includegraphics[width=0.4\linewidth]{bilder/DatasetLocalView2.png} + \caption{Argument Graph from Argument Web} + \end{figure} +\end{frame} + + +\subsection{Benchmark Argument Ranking} +\begin{frame} + \frametitle{Benchmark Argument Ranking} + \begin{itemize} + \item To create the benchmark dataset, Wachsmuth et al. only kept arguments + from the graph that fulfill their requirements + \begin{itemize} + \item If a conclusion was part in more than one argument, it was kept + \item Furhtermore, Wachsmuth et al. removed all nodes that do not contain a + real claim + \item Additionally, an argument: + \begin{itemize} + \item has to be a valid-counter argument + \item must be based on reasonalb premises + \item must allow a logic interference to be drawn + \end{itemize} + \end{itemize} + \end{itemize} +\end{frame} + +\begin{frame} + \frametitle{Benchmark Argument Ranking} + \begin{itemize} + \item The resulting benchmark dataset consist of 32 conclusions which + participated in 110 arguments + \item These 110 arguements were ranked by seven experts from computational + linguistics and information retireval + \item Each argument was ranked by how much each of its premises contributes to + the acceptance or rejection of the conlusion + \end{itemize} +\end{frame} + + +\subsection{Evaluation Method} +\begin{frame} + \frametitle{Evaluation Method} + \begin{itemize} +\item To evaluate the agreement between the experts and to ensure comparability +were than used Kendall's $\tau$ +\item Kendall $\tau$ is correlation coefficient that indicates the agreement +between two quantities with respect to a property +\begin{itemize} +\item In this case, this means the agreement between two experts with respect to +an argument +\item -1 signifies a complete disagreement and +1 a complete agreement +\end{itemize} +\item The mean over all experts for the evaluation of the benchmark is 0.36 + \end{itemize} \end{frame} \ No newline at end of file