KARRIEREWEGE: A Large Scale Career Path Prediction Dataset
Extracted source text · arXiv 2412.14612
% This must be in the first 5 lines to tell arXiv to use pdfLaTeX, which is strongly recommended.
\pdfoutput=1
% In particular, the hyperref package requires pdfLaTeX in order to break URLs across lines.
\documentclass[11pt]{article}
% Change "review" to "final" to generate the final (sometimes called camera-ready) version.
% Change to "preprint" to generate a non-anonymous version with page numbers.
\usepackage{coling}
\usepackage{arydshln}
% Standard package includes
\usepackage{times}
\usepackage{latexsym}
% For proper rendering and hyphenation of words containing Latin characters (including in bib files)
\usepackage[T1]{fontenc}
% For Vietnamese characters
% \usepackage[T5]{fontenc}
% See https://www.latex-project.org/help/documentation/encguide.pdf for other character sets
% This assumes your files are encoded as UTF8
\usepackage[utf8]{inputenc}
% This is not strictly necessary, and may be commented out,
% but it will improve the layout of the manuscript,
% and will typically save some space.
\usepackage{microtype}
% This is also not strictly necessary, and may be commented out.
% However, it will improve the aesthetics of text in
% the typewriter font.
\usepackage{inconsolata}
%Including images in your LaTeX document requires adding
%additional package(s)
\usepackage{graphicx}
%% Text boxes
\usepackage{tcolorbox}
%% Color text
\usepackage{xcolor}
\usepackage{enumitem}
% Importing necessary packages
\usepackage{listings}
\lstset{
inputencoding=utf8,
extendedchars=true,
literate=%
{ä}{{\"a}}1
{ö}{{\"o}}1
{ü}{{\"u}}1
{Ä}{{\"A}}1
{Ö}{{\"O}}1
{Ü}{{\"U}}1
{ß}{{\ss}}1
}
\usepackage{placeins}
\usepackage{float}
\usepackage{afterpage}
% Fixing tables here we go
\usepackage{stfloats}
% Customizing the listings environment
\lstset{
basicstyle=\ttfamily\small, % Use a typewriter font and small font size
breaklines=true, % Enable line breaking
frame=single, % Add a frame around the code
backgroundcolor=\color{gray!10}, % Light gray background for readability
keywordstyle=\color{blue}, % Color for keywords (optional)
commentstyle=\color{gray}, % Color for comments (optional)
showstringspaces=false % Don't show spaces in strings
}
%% Code and name of models
\def\code#1{\texttt{#1}}
% If the title and author information does not fit in the area allocated, uncomment the following
%
%\setlength\titlebox{<dim>}
%
% and set <dim> to something 5cm or larger.
\title{\textsc{Karrierewege}: A Large Scale Career Path Prediction Dataset}
% Author information can be set in various styles:
% For several authors from the same institution:
% \author{Author 1 \and ... \and Author n \\
% Address line \\ ... \\ Address line}
% if the names do not fit well on one line use
% Author 1 \\ {\bf Author 2} \\ ... \\ {\bf Author n} \\
% For authors from different institutions:
% \author{Author 1 \\ Address line \\ ... \\ Address line
% \And ... \And
% Author n \\ Address line \\ ... \\ Address line}
% To start a separate ``row'' of authors use \AND, as in
% \author{Author 1 \\ Address line \\ ... \\ Address line
% \AND
% Author 2 \\ Address line \\ ... \\ Address line \And
% Author 3 \\ Address line \\ ... \\ Address line}
% \author{Elena Senger \\
% MaiNLP, Center for Information and Language Processing, LMU Munich, Germany \\
% Fraunhofer Center for International Management and Knowledge Economy IMW, Germany \\
% \texttt{elena.senger@cis.lmu.de} \\\And
% Yuri Campbell \\
% Fraunhofer Center for International Management and Knowledge Economy IMW, Germany \\
% \texttt{yuri.cassio.campbell.borges@imw.fraunhofer.de} \\\And
% Rob van der Goot \\
% Department of Computer Science, IT University of Copenhagen, Denmark \\
% \texttt{robv@itu.dk} \\\And
% Barbara Plank \\
% MaiNLP, Center for Information and Language Processing, LMU Munich, Germany\\
% \texttt{b.plank@lmu.de}}
\author{Elena Senger$^{1,2}$ \quad Yuri Campbell$^{2}$ \quad Rob van der Goot$^{3}$ \quad Barbara Plank$^{1}$ \\[1.5ex]
$^1$MaiNLP, Center for Information and Language Processing, LMU Munich, Germany \\
$^2$Fraunhofer Center for International Management and Knowledge Economy IMW, Germany \\
$^3$Department of Computer Science, IT University of Copenhagen, Denmark \\
\texttt{elena.senger@cis.lmu.de, yuri.campbell@imw.fraunhofer.de} \\
\texttt{robv@itu.dk, b.plank@lmu.de}
}
%\author{
% \textbf{First Author\textsuperscript{1}},
% \textbf{Second Author\textsuperscript{1,2}},
% \textbf{Third T. Author\textsuperscript{1}},
% \textbf{Fourth Author\textsuperscript{1}},
%\\
% \textbf{Fifth Author\textsuperscript{1,2}},
% \textbf{Sixth Author\textsuperscript{1}},
% \textbf{Seventh Author\textsuperscript{1}},
% \textbf{Eighth Author \textsuperscript{1,2,3,4}},
%\\
% \textbf{Ninth Author\textsuperscript{1}},
% \textbf{Tenth Author\textsuperscript{1}},
% \textbf{Eleventh E. Author\textsuperscript{1,2,3,4,5}},
% \textbf{Twelfth Author\textsuperscript{1}},
%\\
% \textbf{Thirteenth Author\textsuperscript{3}},
% \textbf{Fourteenth F. Author\textsuperscript{2,4}},
% \textbf{Fifteenth Author\textsuperscript{1}},
% \textbf{Sixteenth Author\textsuperscript{1}},
%\\
% \textbf{Seventeenth S. Author\textsuperscript{4,5}},
% \textbf{Eighteenth Author\textsuperscript{3,4}},
% \textbf{Nineteenth N. Author\textsuperscript{2,5}},
% \textbf{Twentieth Author\textsuperscript{1}}
%\\
%\\
% \textsuperscript{1}Affiliation 1,
% \textsuperscript{2}Affiliation 2,
% \textsuperscript{3}Affiliation 3,
% \textsuperscript{4}Affiliation 4,
% \textsuperscript{5}Affiliation 5
%\\
% \small{
% \textbf{Correspondence:} \href{mailto:email@domain}{email@domain}
% }
%}
\begin{document}
\maketitle
\begin{abstract}
Accurate career path prediction can support many stakeholders, like job seekers, recruiters, HR, and project managers. However, publicly available data and tools for career path prediction are scarce.
In this work, we introduce \textsc{Karrierewege}, a comprehensive, publicly available dataset containing over 500k career paths, significantly surpassing the size of previously available datasets. We link the dataset %containing German taxonomy codes
to the ESCO taxonomy to offer a valuable resource for predicting career trajectories. To tackle the problem
of free-text inputs typically found in resumes, we
enhance it by synthesizing
job titles and descriptions resulting in \textsc{Karrierewege+}. This allows for accurate predictions from unstructured data, closely aligning with real-world application challenges. We benchmark existing state-of-the-art (SOTA) models on our dataset and a prior benchmark and observe improved performance and robustness, particularly for free-text use cases, due to the synthesized data.
\end{abstract}
\section{Introduction}
Career path prediction (also known as career trajectory prediction) is a growing field \citep{Shreyas.2024}, with the potential to inform recruitment, career counseling, upskilling or reskilling, or more broadly workforce planning and workforce trends. The task is to predict future career moves based on an individual's work history, possibly using further information such as skills or education. To achieve this, robust datasets that capture detailed career histories are essential. However, the availability of large-scale benchmark career history datasets remains limited \citep{du2024laborllmlanguagebasedoccupationalrepresentations, decorte.2023}, posing a major challenge for the field. % development and comparison of such models.
A dataset mapped to ESCO (European Skills, Competences, Qualifications, and Occupations) Taxonomy is particularly advantageous because ESCO provides a standardized ``common language'' for occupations and skills across the European labor market, describing over 3,000 occupations and nearly 14,000 skills in 28 languages.\footnote{\url{https://esco.ec.europa.eu/en/about-esco/what-esco}}
Since its introduction in 2017, ESCO has attracted diverse stakeholders—including employment services, job portals, educational institutions, HR departments, and international organizations.\footnote{\url{https://esco.ec.europa.eu/en/about-esco/esco-stakeholders}} %This supports applications such as matching job seekers to suitable jobs based on their skills, suggesting training for upskilling or reskilling, and aiding employment agencies in workforce planning.
% ESCO is increasingly recognized internationally alongside O*NET and facilitates interoperability between systems.\
Building on these insights, we release a large, publicly available dataset mapped to ESCO occupations to address the critical need for comprehensive and standardized resources in career path prediction. By leveraging ESCO's ``common language'' for occupations and skills, our dataset aims to foster research and development in this growing field, paving the way for more accurate career trajectory modeling. We document the steps involved in our dataset creation process to encourage the development and evaluation of customized datasets tailored to real-world applications,
% By releasing a large, publicly available dataset mapped to ESCO occupations, we aim to foster research and development on career path prediction, enabling more sophisticated and accurate career path modeling. We document the steps of our dataset creation process, with the aim to encourage the development and evaluation of customized datasets tailored towards real-world applications, %as well as career path prediction models,
ultimately promoting job mobility and fostering a more integrated and efficient labor market.
%In this paper, we aim to address this gap by:
Our contributions are:
\begin{itemize}[itemsep=1pt, topsep=1pt, parsep=1pt, partopsep=1pt]
\item Introducing \textsc{Karrierewege}, a new large-scale career path prediction dataset consisting of over 500,000 career paths.%, making it one of the most extensive publicly available datasets for career trajectory research
\footnote{Karrierewege: \url{https://huggingface.co/datasets/ElenaSenger/Karrierewege}\\ Karrierewege plus: \url{https://huggingface.co/datasets/ElenaSenger/Karrierewege_plus}}
\item Mapping the dataset to the ESCO taxonomy, enhancing interoperability and facilitating research and real-world applications that utilize ESCO. %, such as employment agencies, job matching platforms, and educational institutions.
\item Exploring data synthesis techniques by generating paraphrased titles and descriptions from the taxonomical dataset to address the real-world challenge of free text data.
\item A reproduction study of \citet{decorte.2023} to compare results on their benchmark dataset with the newly introduced \textsc{Karrierewege} and \textsc{Karrierewege+} datasets.
\end{itemize}
\section{Related Work}
\subsection{Datasets for Career Path Prediction}
In the literature on machine learning-based career path prediction, most prior work uses large \textit{non-public} datasets, typically sourced from major career portals such as LinkedIn \citep{Li.2027, CerillaSVF23}, Randstad \citep{schellingerhout2022explainable}, or Zippia \citep{vafa2024careerfoundationmodellabor}. As publicly available datasets, survey data is a popular choice \citep{Chang2019ABA, vafa2024careerfoundationmodellabor, du2024laborllmlanguagebasedoccupationalrepresentations} -- see Table \ref{tab:datasets}. But survey data is typically relatively small, or does not track the same individuals over a longer time span. For example, the Current Population Survey---a national U.S. labor force survey used in \citet{Chang2019ABA}---has a panel of 54,000 respondents per year but contains a person's occupation for only two consecutive years. Other surveys \cite{vafa2024careerfoundationmodellabor,du2024laborllmlanguagebasedoccupationalrepresentations} are relatively small with sizes around 9-12k respondents. A small publicly available dataset is introduced by \citet{decorte.2023}. It is created using a Kaggle dataset of 2,482 anonymized English resumes. All occupations are linked to ESCO (version 1.1.2). The dataset includes both self-written job titles and synthetic descriptions (grounded on resumes), as well as standardized ESCO titles. Inspired by~\citet{decorte.2023}, %Since we are also interested in predicting ESCO occupations, we use this dataset for comparison and
we use their SOTA approach %of the paper for benchmarking.
and compare results to their smaller dataset (see further Section~\ref{sec:datasetcomp}).
\begin{table}[h!]
\centering
\resizebox{\columnwidth}{!}{
\begin{tabular}{p{4.7cm}|p{5.3cm}|r}
\hline
\textbf{Dataset} & \textbf{Paper} & \textbf{Size} \\
\hline
Nat. Longitudinal Survey of Youth 1979 & \citet{vafa2024careerfoundationmodellabor}, \citet{du2024laborllmlanguagebasedoccupationalrepresentations} & 1,200 \\
Nat. Longitudinal Survey of Youth 1997 & \citet{vafa2024careerfoundationmodellabor}, \citet{du2024laborllmlanguagebasedoccupationalrepresentations} & 9,000 \\
Panel Study of Income Dynamics & \citet{vafa2024careerfoundationmodellabor}, \citet{du2024laborllmlanguagebasedoccupationalrepresentations} & 12,000 \\
Current Population Survey* & \citet{Chang2019ABA} & 54,000 \\
DECORTE & \citet{decorte.2023} & 2,000 \\
\textsc{Karrierewege} & our paper & 500,000 \\
\textsc{Karrierewege+} & our paper & 100,000 \\
\hline
\end{tabular}
}
\caption{\label{tab:datasets} Summary of datasets used in various studies. *Only data for two consecutive years per person.}
\end{table}
%They use two cohorts of the National Longitudinal Survey of Youth from 1979 (NLSY79) and 1997 (NLSY97), as well as the Panel Study of Income Dynamics (PSID).
\subsection{Methods for Synthetic Data Generation and LLMs in Occupations}
%here I want to say what my problem is and than that paraphrasing and description generation has been done successfully by LLMs (so it should be a good method for me) and that LLMs have been used for other tasks in the occupation/job market domain, so we know they work well in that domain what also support our method of using a LLM for synthetic data generation. So we will paraphrase the job titles and than create description for each title.
We source the original raw data from the German Employment Agency, but it only includes standardized job titles and descriptions. To make the model more applicable to real-world scenarios, where resumes often use varied, paraphrased job titles, we generated synthetic training data. This allows for more robust career path prediction models that can handle the complexities of free-text inputs.
Off-the-shelf (non-fine-tuned) large language models (LLMs) have been successfully applied to paraphrasing tasks across various domains \citep{Jayawardena_2024} and have also been used to generate synthetic data in the job market, such as to create job vacancies \citep{li2023llm4jobsunsupervisedoccupationextraction, magron-etal-2024-jobskape}. Their effectiveness in representing occupations likely stems from the extensive training of LLMs on diverse sources of data, including occupational data, labor market news and job-related texts \citep{du2024laborllmlanguagebasedoccupationalrepresentations}. This demonstrated success in the occupational domain supports our approach of leveraging LLMs for synthesizing training data by paraphrasing job titles and generating corresponding descriptions.
%\subsection{Methods for Career Path Prediction}
%Career path prediction is a well-established research area with a variety of methodologies. Some studies, such as \citet{Groot2021JobPK} and \citet{Gugnani.2018}, utilize skill and occupation graphs to predict career trajectories. \citet{Chang2019ABA} introduced a Bayesian factorization model for predicting the next occupation. A widely adopted approach in this domain involves LSTM-based architectures~\citep{Li.2027, CerillaSVF23, He.2022, schellingerhout2022explainable}. The CAREER model \citep{vafa2024careerfoundationmodellabor} leverages transformers to iteratively build representations of career histories using attention mechanisms across layers, showing strong performance when pre-trained on large-scale resume datasets and fine-tuned on smaller survey data. These advances reflect the growing role of language models in career prediction. \citet{decorte.2023} used a fine-tuned sentence-transformer model to encode job titles, followed by a linear projection and skill-based scoring to represent career paths. More recently, \citet{du2024laborllmlanguagebasedoccupationalrepresentations} introduced LABOR-LLM, which fine-tunes large language models for next-job prediction, further improving on previous models like CAREER. Given the relatively small size of the dataset introduced by \citet{decorte.2023}, we opted to adopt the method proposed in their paper rather than LSTM-based approaches, which typically benefit from larger datasets.%reason why we didn't finetune lama?
\section{\textsc{Karrierewege}}
\begin{table*}
\centering
\resizebox{2\columnwidth}{!}{
\begin{tabular}{c|c|p{4.1cm}|p{4cm}|p{4cm}|p{4cm}|p{4cm}|p{4cm}}
\hline
\textbf{id} & \textbf{order} & \textbf{ESCO\_title} & \textbf{ESCO\_description} & \textbf{new\_title\_oc} & \textbf{new\_description\_oc} & \textbf{new\_title\_cp} & \textbf{new\_description\_cp} \\
\hline
0 & 0 & Medical laboratory manager & Medical laboratory managers oversee ... & Quality Assurance Specialist & The Quality Assurance Specialist is ... & Laboratory Director & Laboratory Director: Oversees ... \\
\hline
0 & 1 & Environmental health inspector & Environmental health inspectors carry ... & Pollution Control Specialist & The Pollution Control Specialist is responsible ... & Environmental Safety Specialist & Environmental Safety Specialist: Conducts ... \\
\hline
0 & 2 & Environmental health inspector & Environmental health inspectors carry ... & Environmental Compliance Officer & Environmental Compliance Officer: Conducts ... & Environmental Safety Specialist & Environmental Safety Specialist: Conducts ... \\
\hline
1 & 0 & Food service worker & Food service workers prepare food and ... & Concession Stand Staff & Operate a concession stand, selling ... & Culinary Service Provider & Culinary Service Provider: Provides expert culinary ... \\
\hline
1 & 1 & Dietitian & Dietitians assess specific nutritional ... & Eating Disorder Specialist & The Eating Disorder Specialist provides ... & Nutritionist & Nutritionist: Helps people develop healthy ... \\
\hline
\end{tabular}
}
\caption{\label{tab:job_mapping_simplified} Example entries from the \textsc{Karrierewege+} validation dataset. Rows sharing the same \textit{id} refer to work experiences of the same person, with \textit{order} indicating the sequence. Titles and descriptions with the \textit{\_oc} suffix are synthesized per occupation, while those with \textit{\_cp} are synthesized per career path.}
\end{table*}
\begin{figure*}
\centering
\includegraphics[width=1.0\linewidth]{Karrierewege_new_small.png}
\caption{Steps necessary to create \textsc{Karrierewege} and \textsc{Karrierewege+} datasets. Titles and descriptions with the \textit{\_oc} suffix are synthesized per occupation, while those with \textit{\_cp} are synthesized per career path.}
\label{fig:datasets}
\end{figure*}
To create a large and diverse dataset for career path prediction, we sourced the data from anonymized resumes provided by the German employment agency as a basis (see Figure \ref{fig:datasets} for the dataset creation process).\footnote{\url{https://www.arbeitsagentur.de/bewerberboerse/}} This dataset encompasses resumes from individuals seeking employment across all industries. We note that despite of its size, the resulting dataset may still be biased---it possibly contains more resumes from industries with lower demand (where individuals are more inclined to register as unemployed) than from high-demand industries, where unemployment registration is less common. Additionally, since all resumes are from individuals seeking employment in Germany, there is a cultural bias towards that region. %when applying this dataset to other regions.
\subsection{Mapping to ESCO}
Due to restrictions preventing the direct publication of these anonymized CVs, and in recognition of the widespread adoption of the ESCO framework, we manually mapped occupations from the German resumes to their equivalents in the German ESCO taxonomy (version 1.2.0). This mapping ensures compatibility with the widely adopted ESCO framework, enriches the dataset with additional attributes like skills and job descriptions, and enables accessibility in 28 languages. For consistency with previous work, we use the English ESCO attributes in this paper. Yet, the published dataset can be converted to any of the other languages via the unique identifiers provided by ESCO.
The raw dataset contains standardized occupational titles from the German Berufenet taxonomy (version 2020).\footnote{\url{https://web.arbeitsagentur.de/berufenet/}} Both Berufenet and ESCO occupations are mapped to ISCO-08 codes. However, due to broad categorization inherent in ISCO-08, multiple occupations share the same code, making it unsuitable for direct one-to-one mapping.
%\footnote{ESCO to ISCO-08 mapping: \url{https://esco.ec.europa.eu/en/about-esco/escopedia/escopedia/international-standard-classification-occupations-isco}; Berufenet to ISCO-08 mapping: \url{https://statistik.arbeitsagentur.de/DE/Statischer-Content/Grundlagen/Klassifikationen/Klassifikation-der-Berufe/KldB2010-Fassung2020/Arbeitsmittel/Umschluesselungstabellen.html}}
%Berufenet includes occupations, training programs, degrees, and supplementary information such as descriptions and required skills.
Therefore, to link Berufenet and ESCO, we experimented with three methods as outlined next.
% However, ultimately, none of these automated method produced consistently satisfactory results, so we used them only to speed up the manual linkage process conducted by a trained assistant and one of the authors. %The first method involved embedding similarity, using the ``sentence-transformers/distiluse-base-multilingual-cased-v2'' model to calculate similarity based on job titles and descriptions. The second method utilized the ESCO API, which returned less accurate mappings due to incomplete queries and higher-level job titles. The third method involved GPT models, specifically GPT-3.5 and GPT-4o-mini for ranking mappings, with the latter yielding the best results, particularly when using English prompts.
% begin{itemize}[itemsep=0.5pt, topsep=0.5pt]
% \item \textbf{Embedding Similarity}: Using the \code{distiluse-base-multilingual-cased-v2}\footnote{\url{https://huggingface.co/sentence-transformers/distiluse-base-multilingual-cased-v2}} model to calculate similarity based on job titles and descriptions.
% \item \textbf{ESCO API}: Querying the ESCO API, which returned less accurate mappings due to incomplete queries and higher-level job titles.
% \item \textbf{GPT Models}: Using \code{GPT-3.5/GPT-4o-mini} for ranking mappings; the latter yielded the best results, especially with English prompts.
% \end{itemize}
% %Given the limited context window of the model, we employed ISCO-08 codes as a filtering mechanism to narrow down candidate occupations for the GPT models (for detailed prompts see Appendix \ref{sec:prompts-linkage}).
% Overall, \code{GPT-4o-mini} performed best, achieving around 60\% correct mappings (see Table \ref{tab:embedding_similarity}).
%\code{-cased-v2}\footnote{\url{https://huggingface.co/sentence-transformers/distiluse-base-multilingual-cased-v2}}
The first method involved embedding similarity, using the \code{distiluse-base-multilingual} model to calculate semantic similarity based on job titles and descriptions. The second method utilized the ESCO API, which returned less accurate mappings due to incomplete queries and higher-level job titles. The third method involved GPT models, specifically \code{GPT-3.5} and \code{GPT-4o-mini} for ranking mappings.
Overall, \code{GPT-4o-mini} performed best, particularly when using English prompts, achieving around 60\% correct mappings (see Table \ref{tab:embedding_similarity}). However, ultimately, none of these approaches consistently produced satisfactory results. Hence, we used them only to speed up the manual linkage process conducted by a trained assistant and one of the authors.
%Due to restrictions preventing the direct publication of these anonymized CVs, and in recognition of the widespread adoption of the ESCO framework in academic literature, we manually linked the occupations from the German resumes to their corresponding entries in the German version of the ESCO taxonomy (version 1.2.0). This mapping not only ensures compatibility with the established ESCO framework but also enriches the dataset with additional attributes such as skills and job descriptions. Moreover, because the ESCO taxonomy is available in 28 languages, this mapping enables the dataset to be accessible in all these languages, rather than being limited to German. In this paper, we use the English ESCO attributes to ensure fair comparisons with previous work. This English-transformed dataset is what we publish, but conversions to any of the other 27 languages are possible via the unique identifiers provided by the ESCO taxonomy.
%The raw dataset contains standardized occupational titles from the German Berufenet taxonomy (version 2020).\footnote{\url{https://web.arbeitsagentur.de/berufenet/}} The Berufenet taxonomy includes occupations, training programs, and degrees, as well as supplementary information such as descriptions and required skills. Both the occupations in the Berufenet taxonomy and those in the ESCO taxonomy are mapped to ISCO-08 codes.\footnote{ESCO to ISCO-08 mapping: \url{https://esco.ec.europa.eu/en/about-esco/escopedia/escopedia/international-standard-classification-occupations-isco}; Berufenet to ISCO-08 mapping: \url{https://statistik.arbeitsagentur.de/DE/Statischer-Content/Grundlagen/Klassifikationen/Klassifikation-der-Berufe/KldB2010-Fassung2020/Arbeitsmittel/Umschluesselungstabellen.html}} However, due to the broad categorisation inherent in the ISCO-08 classification, several occupations within either the ESCO or Berufenet taxonomies share the same ISCO-08 code. Consequently, the ISCO-08 code acts primarily as a filtering mechanism rather than facilitating a direct one-to-one mapping between Berufenet and ESCO.
%In order to achieve an accurate linkage between the Berufenet and ESCO taxonomies, we experimented with several methods. We employed embedding similarity using the ``sentence-transformers/distiluse-base-multilingual-cased-v2'' model to calculate similarity scores based on either job titles or job descriptions. Furthermore, we used the ESCO API and tested two GPT models to facilitate automatic linkage. Given the limited context window of the model, we employed ISCO-08 codes as a filtering mechanism to narrow down candidate occupations. The prompt provided to the GPT models included the original Berufenet title, the results from the three previous methods (embedding similarity based on titles and descriptions, and the ESCO API), and the candidate occupations derived from ISCO-08 blocking.
%The most successful approach was to use GPT-4o-mini to rank potential mappings, see \ref{tab:embedding_similarity}. The use of English prompts significantly improved the mapping of correctly linked occupations. For the prompts used see Appendix \ref{sec:prompts-linkage}.
%The semantic similarity methods linked around 50 percent of Berufenet occupations correctly to their ESCO equivalents. The ESCO API method proved to be the least effective. One significant limitation was that the API returned results for only 83 percent of the queries. In addition, the API occasionally returned more abstract or higher-level job or industry titles, resulting in less accurate mappings. All scores in the Table \ref{tab:embedding_similarity} are calculated based on this manually linked gold standard.
%Overall, none of these automated methods produced consistently satisfactory results, and were therefore only used to speed up the manual linkage process. The linkage was carried out by a hired and trained student assistant and one of the authors.
\begin{table}
\resizebox{\columnwidth}{!}{
\centering
\begin{tabular}{cccc}
\hline
Method & \% Correct Links \\
\hline
Embedding Similarity Title & 51.1 \\
Embedding Similarity Description & 51.2 \\
ESCO API & 30.7 \\
GPT 3.5 DE Prompt & 42.6 \\
GPT 3.5 EN Prompt & 52.9 \\
GPT 4o mini DE Prompt & 59.4 \\
GPT 4o mini EN Prompt & 60.4 \\
\hline
\end{tabular}
}
\caption{\label{tab:embedding_similarity}
Percentage of correct links per method for mapping ESCO and Berufenet occupations.
}
\end{table}
\subsection{Filtering the Data}
We excluded all resumes with missing entries in the work history field and kept only those resumes with more than one and less than thirty work experiences. We also kept only resumes with a change of occupation in their career history, as we are particularly interested in learning and predicting these. Additionally, we excluded resumes that contained rare occupations (less than 10 times in the dataset). This resulted in 568,888 resumes.
\section{\textsc{Karrierewege+}: Synthesized Data}
\subsection{Generating Free-Text Job Titles}
%To enable models to process the varied and free-text data typically found in resumes, we developed synthetic training data through two data synthesis methods, enhancing the model's ability to handle real-world applications and improving its robustness for career path prediction.
To generate free-text data, we use two data synthesizes methods:
\paragraph{\textsc{Karrierewege+}oc}
In the first approach, we use \code{LLAMA 3.1 8b} to generate seven alternative titles for each ESCO occupation title (K+oc). The choice of seven paraphrased titles was based on empirical observations indicating that generating more than seven titles often resulted in lower quality titles. For each paraphrased title, we additionally generated a corresponding job description using the same model. The underlying hypothesis for this method is that the paraphrased titles remain closely related to the original titles while being sufficiently distinct from other ESCO titles.
\paragraph{\textsc{Karrierewege+}cp}
In the second approach, we directly synthesized the entire sequence of titles of a career path (K+cp). The hypothesis guiding this approach is that providing the model with the context of previous and subsequent occupation titles enables it to generate more appropriate and contextually relevant paraphrased titles. This method aims to achieve higher diversity by paraphrasing each ESCO occupation title more frequently, thereby introducing slight variations and increasing the richness of the dataset. Similar to the first approach, a corresponding job description was generated for each paraphrased title. The language model used was again the \code{LLAMA 3.1 8b} model. Due to the computational intensity of synthesizing individual career paths, we limited this approach to a random subset of 100,000 resumes. To maintain comparability between the two synthesis methods, we restricted the first synthesizing approach to the same number of resumes. All prompts are provided in Appendix \ref{sec:prompts-generation}.
\subsection{Evaluating the Quality of Paraphrased Titles and Career Paths}
\subsubsection{Quantitative Analysis}
To evaluate the quality of the paraphrased job titles and descriptions, we followed best practices recommended by \citet{van-der-lee-etal-2019-best} for paraphrase evaluation, i.e., to use well-defined evaluation criteria, avoid the use of smaller scales in rating (e.g., 2-point or 3-point Likert scale), employing a within-subjects design (where evaluators reviewed outputs from all systems), randomized orderings to mitigate bias from order effects and complement subjective with objective measures to provide a comprehensive evaluation. Following \citet{Jayawardena_2024}, we used BLEU \citep{papineni-etal-2002-bleu}, ROUGE-L \citep{lin-2004-rouge}, and a 5-point Likert scale to evaluate the quality of the paraphrased job titles. We overall assessed the generated paraphrases on four key dimensions:
\begin{itemize}[itemsep=0.5pt, topsep=1pt]
\item \textbf{Correctness}: Measures if paraphrased titles are valid job titles and distinct from the original. Scores range from 0 (invalid or identical titles) to 5 (all titles valid and distinct).
\item \textbf{Semantic Similarity}: Evaluates how well paraphrased titles capture the meaning of the original titles. Scores range from 0 (low similarity) to 5 (high similarity).
\item \textbf{Diversity}: Assesses variety in the paraphrased career paths with a score of 0 indicating repetition, while 5 reflects a wide range of titles.
\item \textbf{Coherence}: Measures the logical coherence of the paraphrased titles with the career path, where 0 means titles do not form a logical progression, and 5 indicates a coherent sequence.
\end{itemize}
To evaluate these dimensions, we manually labeled 100 resumes. One author, unaware of which synthesis method was used, manually evaluated each resume on the four dimensions. To further validate our findings, we used \code{GPT-4o mini} to evaluate the same metrics, after checking the alignment on the 100 manually labeled samples. We experimented with two prompt versions: one where the model was prompted once for all metrics, and another where the model was prompted for each metric individually (see the Appendix \ref{sec:prompts-evaluation} for the prompts). Following the best practice of \citet{thakur2024judgingjudgesevaluatingalignment}, we used Cohen’s kappa as a measure of alignment. Cohen’s kappa and the mean values for each metric indicated that prompting the model for all metrics at once resulted in closer alignment with human judgments. In general, Cohen’s kappa values were relatively low, particularly for coherence, but showed stronger alignment for less subjective metrics like diversity and correctness (see Appendix \ref{sec:appendix_alignement_scores}
for detailed scores).
The human and LLM scores revealed that K+cp achieved higher mean scores in correctness, semantic similarity, and coherence compared to K+oc. However, the K+oc outperformed K+cp in terms of diversity. These results are consistent with our expectations: the K+cp processes the entire career path, allowing for more coherent title generation, while the K+oc tends to produce greater diversity since it randomly selects from seven paraphrased options for each occupation. Overall, the Likert scale scores suggest that the K+cp yields higher-quality paraphrases.%compute scores for LLM one prompt for whole dataset (still running)
As objective complementary measures, we used BLEU and ROUGE-L, comparing sequences of job titles and descriptions across entire career paths. For BLEU, we applied a smoothed score to account for low n-gram overlaps. In both metrics, K+cp consistently achieved slightly higher values, indicating better lexical similarity and sequence alignment with the original labels. However, the overall low scores suggest notable differences between both methods and the original career path (see Appendix \ref{sec:appendix_alignement_scores}
for detailed scores).
\subsubsection{Qualitative Analysis}
\begin{table*}
\centering
\resizebox{2\columnwidth}{!}{
\begin{tabular}{l|l|l}
\hline
\textbf{Category} & \textbf{Original Job Title} & \textbf{Paraphrased Job Title} \\
\hline
\textbf{1. Increasing Professionalism}
& Cashier & Retail Sales Associate \\
& Mover & Professional Mover \\
& Bricklayer & Building Specialist \\
\hline
\textbf{2. Inaccurate or Non-existent Titles} & Technical Communicator & User Experience \\
& Financial Broker & Wealth Manager: Helping clients... \\
& Bicycle Courier & On-the-Go Logistics Professional \\
%& House Sitter & Domestic Companion \\
%& Postman/Postwoman & Letter Delivery Person \\
& Printed Circuit Board Assembler & PCB \\
\hline
\textbf{3. Semantic Mismatch} & City Councillor & Urban Planner \\
& House Sitter & Home Care Provider \\
& Food Production Operator & Production Line Worker \\
& Foster Care Support Worker & Support Worker \\
\hline
\textbf{4. Career Path as Story} & Hairdresser $\rightarrow$ Sales Account Manager $\rightarrow$ & Beauty Professional $\rightarrow$ Business Developer $\rightarrow$ \\
& Commercial Sales Rep $\rightarrow$ Hairdresser & Sales Specialist $\rightarrow$ Salon Owner/Operator \\
\cline{2-3} % Horizontal line to separate two career paths
& Vehicle Cleaner $\rightarrow$ Factory Hand $\rightarrow$ & Construction Laborer $\rightarrow$ Flooring Installer $\rightarrow$ \\
& Metal Sawing Machine Operator & Floor Covering Technician \\
\hline
\end{tabular}
}
\caption{\label{tab:qualitative} Examples of qualitative analysis of paraphrased job titles}
\end{table*}
The paraphrased job titles reveal several interesting and distinct trends and errors. One common trend is the enhancement of professionalism, where paraphrased titles elevate the perceived professionalism of the original job titles, like ``Bricklayer'' becomes ``Building Specialist''. Another issue is the introduction of inaccurate or non-existent job titles, such as ``On-the-Go Logistics Professional''
where the paraphrasing becomes overly creative and diverges from widely recognized titles. Semantic mismatches also occur, for instance, when “city councillor,” a political position, is paraphrased as “urban planner,” a technical role focused on infrastructure. Lastly, when using the K+cp, the paraphrasing often constructs cohesive career paths, showing clear progression and skills development over time. However, when paraphrasing individual occupations without considering the full career path, this cohesive narrative is lost. Examples of these patterns are presented in Table \ref{tab:qualitative}.
%\paragraph{1. Increasing Professionalism:} In many cases, the paraphrased titles elevated the perceived professionalism of the original job titles:
%\begin{itemize}[noitemsep, topsep=1pt]
% \item \textit{Original}: ['contact centre supervisor', 'cashier', 'mover']
% \item \textit{Paraphrased}: ['Customer Service Leader', 'Retail Sales Associate', 'Professional Mover']
%\end{itemize}
%The paraphrased titles convey a more formal and polished tone, making the occupations sound more professional. Similarly, a common manual labor job such as \textit{bricklayer} was paraphrased as \textit{Building Specialist}, enhancing the perceived expertise associated with the role.
%\paragraph{2. Inaccurate or Non-existent Occupations:} Some paraphrases deviated significantly from real-world job titles or became overly creative, resulting in inaccurate representations:
%\begin{itemize}[noitemsep, topsep=3pt]
% \item \textit{Original}: \textit{technical communicator}
% \item \textit{Paraphrased}: \textit{User Experience}
%\end{itemize}
%In this case, 'User Experience' is no occupation title.
%\begin{itemize}[noitemsep, topsep=3pt]
% \item \textit{Original}: \textit{financial broker}
% \item \textit{Paraphrased}: \textit{Wealth Manager: Helping clients manage their wealth and investments}
%\end{itemize}
%The paraphrase introduces unnecessary details.
%\begin{itemize}[noitemsep, topsep=3pt]
% \item \textit{Original}: \textit{bicycle courier}
% \item \textit{Paraphrased}: \textit{On-the-Go Logistics Professional}
%\end{itemize}
%This paraphrase takes a more creative route but may stray too far from common job titles. Similarly, \textit{house sitter} turned into \textit{Domestic Companion} or \textit{postman/postwoman} turned into \textit{Letter Delivery Person} , which are not a recognized job titles.
%\begin{itemize}[noitemsep, topsep=3pt]
% \item \textit{Original}: \textit{printed circuit board assembler}
% \item \textit{Paraphrased}: \textit{PCB}
%\end{itemize}
%This is no paraphrase.
%\paragraph{3. Issues with Semantic Similarity:} Certain paraphrases suffered from low semantic similarity, where the new title did not adequately reflect the original meaning:
%\begin{itemize}[noitemsep, topsep=3pt]
% \item \textit{Original}: \textit{city councillor}
% \item \textit{Paraphrased}: \textit{Urban Planner}
%\end{itemize}
%A 'city councillor' is a political position, while 'urban planner' is a technical role focused on city infrastructure, illustrating a mismatch in meaning.
%\begin{itemize}[noitemsep, topsep=3pt]
% \item \textit{Original}: \textit{house sitter}
% \item \textit{Paraphrased}: \textit{Home Care Provider}
%\end{itemize}
%This paraphrase confuses the role of someone who watches over a house with a care provider, which usually implies personal or medical care.
%\begin{itemize}[noitemsep, topsep=3pt]
% \item \textit{Original}: \textit{food production operator}
% \item \textit{Paraphrased}: \textit{Production Line Worker}
%\end{itemize}
%\begin{itemize}[noitemsep, topsep=3pt]
% \item \textit{Original}: \textit{foster care support worker}
% \item \textit{Paraphrased}: \textit{support worker}
%\end{itemize}
%Here, the paraphrases are overly vague, omitting the specific context of foster care, which is critical to the job's description.
%\paragraph{4. Constructing a Career Path as a Story:} When looking at entire career paths, the paraphrased titles can sometimes form a narrative, showing progression or cohesion:
%\begin{itemize}[noitemsep, topsep=3pt]
% \item \textit{Original career path}: ['hairdresser', 'sales account manager', 'commercial sales representative', 'commercial sales representative', 'commercial sales representative', 'commercial sales representative', 'hairdresser']
% \item \textit{Paraphrased career path}: ['Beauty Professional', 'Business Developer', 'Sales Specialist', 'Sales Expert', 'Sales Executive', 'Sales Director', 'Salon Owner/Operator.']
%\end{itemize}
%In this example, the paraphrased titles create a more coherent and progressive story, reflecting a natural career advancement from a beauty professional to a business-focused sales expert and eventually an owner/operator.
%Another example of this storytelling effect can be seen in:
%\begin{itemize}[noitemsep, topsep=3pt]
% \item \textit{Original career path}: ['vehicle cleaner', 'factory hand', 'factory hand', 'metal sawing machine operator']
% \item \textit{Paraphrased career path}: ['Construction Laborer', 'Carpentry Assistant', 'Floor Layer and Finisher', 'Flooring Installer']
% \item \textit{Paraphrased by occupation}:['Site Laborer', 'Building Maintenance Specialist', 'Flooring Installer', 'Floor Covering Technician']
%\end{itemize}
%This sequence constructs a clear career progression in the construction and flooring industry, creating a logical narrative of skills development over time. However, when paraphrasing by individual occupation rather than considering the whole career path, no such cohesive narrative is formed.
\section{Dataset Statistics and Comparison}\label{sec:datasetcomp}
To ensure the practical utility of \textsc{Karrierewege} for real-world career path prediction, we present key statistics on the dataset's characteristics—including the number of resumes, the distribution and diversity of ESCO occupations and industries, and statistics on synthesized job titles and descriptions that reflect the complexity of free-text inputs. These insights demonstrate the dataset's comprehensiveness and its suitability for advancing both academic research and industrial applications.
%To ensure the utility of \textsc{Karrierewege} for career path prediction in real-world scenarios, we provide an overview of the dataset's characteristics. This section outlines key statistics, including the number of resumes, the distribution and diversity of ESCO occupations, and the diversity of industries.
%Moreover, while mapping the dataset to the ESCO taxonomy enables interoperability across labor markets, we provide statistics on the synthesized job titles and descriptions, which reflect the complexity of free-text inputs in resumes. These statistics demonstrate the dataset’s comprehensiveness and suitability for advancing both academic research and industrial applications.
\begin{figure}
\centering
\includegraphics[width=1.0\linewidth]{career_path_len_histos_normal_numbers.pdf}
\caption{Work experiences per resume for the \textsc{Karrierewege} and DECORTE dataset.}
\label{fig:jobs_per_resume_our_data}
\end{figure}
\begin{figure}
\centering
\includegraphics[width=1.0\linewidth]{esco_code_value_tree_maps.pdf}
\caption{Tree maps on ESCO codes with one digit.}
\label{fig:datasets_tree_map}
\end{figure}
%%%%%%%%%%% Moved to appendix
\begin{figure}
\centering
\includegraphics[width=0.8\linewidth]{esco_code_value_rank_plot.pdf}
\caption{Rank plot of normalized frequencies of ESCO codes with full digits.}
\label{fig:datasets_rank_plot}
\end{figure}
%%%%%%%%%
On the number and average length of resumes, the \textsc{Karrierewege} dataset (568,888 resumes) contains a higher proportion of resumes with fewer work experiences compared to DECORTE (2,482 resumes), which typically includes five experiences per resume (see Figure \ref{fig:jobs_per_resume_our_data}).
While on the distribution and diversity of occupations, \textsc{Karrierewege} features 1,295 unique ESCO occupations, whereas DECORTE has 1,102.
If aggregated by industry using ESCO taxonomy codes, as shown in Figure \ref{fig:datasets_tree_map}, \textsc{Karrierewege} covers broader economic sectors comprehensively, such as Elementary Occupations (Sector 9) and Service Workers (Sector 5), while DECORTE is concentrated in knowledge workers and managerial activities (Sectors 2 and 1).
Moreover, a rank distribution plot in Figure \ref{fig:datasets_rank_plot} of ESCO occupations further highlights these distinctions: \textsc{Karrierewege}'s most frequent occupations are in Sectors 9, 5, and 8, while DECORTE's are concentrated in Sectors 2 and 1.
Nevertheless, despite these differences in the top occupations, both datasets show similar relative coverage of less frequent occupations, as clearly shown in the tails of both rank distributions.
When considering absolute frequencies, however, \textsc{Karrierewege} exhibits broader overall sector coverage, even within Sectors 1 and 2. We provide a throughout presentation of these frequencies in Appendix \ref{sec:esco_occupations_concentration} and the full first level ESCO classification names in Appendix \ref{sec:esco_first_level} for completeness.
% We provide the full first level ESCO classification names in Appendix \ref{sec:esco_first_level} for completeness.
% and, in Appendix \ref{sec:esco_occupations_concentration}, a throughout assessment of ESCO occupations' concentration.
% A rank distribution plot in Figure \ref{fig:datasets_rank_plot} highlights the concentration of elementary and service-related occupations in \textsc{Karrierewege}, with top occupations from Sectors 9, 5, and 8, while DECORTE’s top occupations are largely managerial and knowledge-based (Sectors 2 and 1). The tail of the distribution is comparable, but \textsc{Karrierewege} shows broader sector coverage, particularly in Sectors 1 and 2, with higher overall frequencies.
Regarding the statistics on the synthetic data, Figure \ref{fig:datasets_syn_len} reveals that the lengths of job titles and descriptions differ depending on the data synthetization method, K+oc or K+cp.
Though differences in job title lengths are minimal, apart from some outliers; K+oc generates consistently longer job descriptions (up to 800 characters), while K+cp produces shorter descriptions, with most under 400 characters. In comparison, ESCO descriptions have comparable length to the K+cp descriptions, while being in general also shorter than the ones generated with method K+oc. %As before, the length of the synthetic titles do not differ substantially from the ESCO titles.
\begin{figure}
\centering
\includegraphics[width=1.0\linewidth]{job_title_description_length_box_plots_esco.pdf}
\caption{Length of generated job titles and job descriptions with both strategies K+oc and K+cp in comparison with ESCO.}
\label{fig:datasets_syn_len}
\end{figure}
\section{Benchmark Baseline}
To showcase how well \textsc{Karrierewege} supports existing models in realistic career prediction, we adapted a SOTA approach \citep{decorte.2023} to benchmark its performance comprehensively. We want to showcase the effect of the dataset size and the robustness of models trained on \textsc{Karrierewege+} in comparison to the smaller, less diverse DECORTE benchmark dataset.% with our data.
% To evaluate the effectiveness of career path prediction using our \textsc{Karrierewege} dataset, we adapted the SOTA approach by \citet{decorte.2023}.
%, ensuring that the method is able to handle the nature of resumes in the pure \textsc{Karrierewege} format, which is the structured ESCO job roles and descriptions.
%For completeness, we apply the original approach of \citet{decorte.2023} for the cases in which structured ESCO data is provided in tandem with unstructured data of resumes. In what follows, we present these two strategies for career path prediction and their benchmark baselines on our dataset in comparison with the state of the art.
\subsection{Method}
We follow the scheme called ``LAST'' in \citet{decorte.2023}, and fine-tune a \code{all-mpnet-base-v2}
% \footnote{\url{https://huggingface.co/sentence-transformers/all-mpnet-base-v2}}
sentence-transformer model using contrastive representation learning on pairs of text documents: one for the career path $ex_1, \cdots, ex_N$ and another for the ESCO occupation $occ_N$. In \citet{decorte.2023}, each work experience $ex_i$ in a career path is represented as:
\vspace{-.3cm}
\begin{tcolorbox}[colback=gray!20, colframe=white, width=\columnwidth, boxrule=0pt]
role: <title in free-form text> \\
description: <description in free-form text>
\end{tcolorbox}
\vspace{-.3cm}
\hspace{-.4cm}and the career path document is composed by concatenating them with a separator token.
In our adaptation, while the career document is composed similarly, a career experience $ex_i$ is represented as:
\vspace{-.7cm}
\begin{tcolorbox}[colback=gray!20, colframe=white, width=\columnwidth, boxrule=0pt]
esco role: <esco occupation title> \\
description: <esco occupation description>
\end{tcolorbox}
\vspace{-.3cm}
\hspace{-.4cm}In turn, on both approaches, the occupation document is structured always as the latter and contains data from ESCO occupation $occ_N$.
Finally, a linear transformation $T$ is learned by minimizing the least squares error between transformed representations of career paths $ex_1, \cdots, ex_{N-1}$ and representations of their next ESCO occupations $occ_N$. Therefore, a career path prediction over the next occupation is achieved by the scoring function naturally induced by the cosine similarity between a transformed career path representation and all ESCO occupation representations.
While \citet{decorte.2023} include further an ESCO skill overlap component in the scoring function, we opt to leave this out, in order to better measure the impact of only using ESCO data and synthetic free text data in our experiments.
Our full experimental setup is given in Appendix \ref{sec:exp_setup}.
% Figure \ref{fig:rep_learning} illustrates the fine-tuning strategy.
% We follow the scheme called ``LAST'' in \cite{decorte.2023} which fine-tunes the \code{all-mpnet-base-v2}\footnote{\url{https://huggingface.co/sentence-transformers/all-mpnet-base-v2}} sentence-transformer model using contrastive representation learning on pairs of text documents: one for the career path and another for the next occupation. In \cite{decorte.2023}, each work experience $ex_i$ in a career path is represented as:
% \begin{tcolorbox}[colback=gray!20, colframe=white, width=\columnwidth, boxrule=0pt]
% role: <title in free-form text> \\
% description: <description in free-form text>
% \end{tcolorbox}
% % \begin{tcolorbox}[colback=gray!20, colframe=white, width=\columnwidth, boxrule=0pt]
% % role: <self-written title> \\
% % description: <synthetic description>
% % \end{tcolorbox}
% \hspace{-.4cm}and the career path document \( \textit{doc\_occ} \) is composed by concatenating them with a separator token `<SEP>`.
% In our adaptation, a career experience $ex_i$ is represented as:
% \begin{tcolorbox}[colback=gray!20, colframe=white, width=\columnwidth, boxrule=0pt]
% esco role: <esco occupation title> \\
% description: <esco occupation description>
% \end{tcolorbox}
% \hspace{-.4cm}and we omit the last experience $ex_N$ in the career path document.
% % In both cases, the corresponding experiences are concatenated with a separator token `<SEP>` to form a single document, denoted as \( \textit{doc\_car} \), the career document.
% In turn, the occupation document, denoted as \( \textit{doc\_occ} \), is structured always as the latter and contains data from only one ESCO occupation, here denoted as $occ_N$, which corresponds to the last experience $ex_N$, this being either self-written or ESCO data.
% Finally, a linear transformation $T$ is learned by minimizing the least squares error between transformed career path representations $T(\mathbf{v}_{C_{1:N-1}})$ and next ESCO occupation representations $\mathbf{v}_{occ_N}$. A career path prediction over the next occupation is achieved by the scoring function naturally induced by the cosine similarity between $T(\mathbf{v}_C)$ and all ESCO occupation representations. The experimental setup can be found in the Appendix \ref{sec:exp_setup}.
\subsection{Results}
\begin{table*}[t!]
\centering
\resizebox{2\columnwidth}{!}{
\begin{tabular}{l|ccc|ccc|ccc}
\hline
\textbf{Metric} & \multicolumn{3}{c|}{\textbf{MRR}} & \multicolumn{3}{c|}{\textbf{R@5}} & \multicolumn{3}{c}{\textbf{R@10}} \\
\hline
\textbf{Train/ Test} & \textbf{DECORTE} & \textbf{\textsc{K+}oc} & \textbf{\textsc{K+}cp} & \textbf{DECORTE} & \textbf{\textsc{K+}oc} & \textbf{\textsc{K+}cp} & \textbf{DECORTE} & \textbf{\textsc{K+}oc} & \textbf{\textsc{K+}cp} \\
\hline
\textbf{DECORTE} & \textbf{0.2427} & 0.1339 & 0.1588 & \textbf{0.3418} & 0.2005 & 0.2302 & \textbf{0.4151} & 0.2669 & 0.3076 \\
\textbf{\textsc{K+}oc} & \underline{0.1303} & \textbf{0.4312} & \underline{0.3784} & 0.2164 & \textbf{0.5340} & 0.4899 & 0.3091 & \textbf{0.6165} & \underline{0.5784} \\
\textbf{\textsc{K+}cp} & 0.1294 & \underline{0.3685} & \textbf{0.4281} & \underline{0.2186} & \underline{0.4693} & \textbf{0.5280} & \underline{0.3235} & \underline{0.5566} & \textbf{0.6065} \\
\hline
\end{tabular}
}
\caption{\label{tab:combined_results}
Cross evaluation results for MRR, R@5, and R@10 across free-text datasets.
}
\end{table*}
% \begin{table}[h!]
% \centering
% \resizebox{\columnwidth}{!}{
% \begin{tabular}{l|ccc}
% \hline
% \textbf{Dataset} & \textbf{MRR} & \textbf{R@5} & \textbf{R@10} \\
% \hline
% \textbf{DECORTE 2k} & 0.2427 & 0.3418 & 0.4151 \\
% \hdashline
% \textbf{\textsc{K+}oc 2k} & 0.3846 & 0.4779 & 0.5423 \\
% \textbf{\textsc{K+}oc 100k} & \textbf{0.4312} & \textbf{0.5340} & \textbf{0.6165} \\
% \hdashline
% \textbf{\textsc{K+}cp 2k} & 0.3702 & 0.4754 & 0.5568 \\
% \textbf{\textsc{K+}cp 100k} & \textbf{0.4281} & \textbf{0.5280} & \textbf{0.6065} \\
% \hline
% \textbf{DECORTE ESCO 2k} & 0.2084 & 0.2813 & 0.3418 \\
% \hdashline
% \textbf{\textsc{Karrierewege} 2k} & 0.4232 & 0.5146 & 0.5636 \\
% \textbf{\textsc{Karrierewege} 100k} & 0.4775 & 0.5671 & 0.6317 \\
% \textbf{\textsc{Karrierewege} 500k} & \textbf{0.4867} & \textbf{0.5713} & \textbf{0.6347} \\
% \hline
% \end{tabular}
% }
% \caption{\label{tab:results}
% Results for different free-text and standardized resume datasets with their approximate size.
% }
% \end{table}
% \begin{table*}[b]
% \resizebox{2\columnwidth}{!}{
% \centering
% \begin{tabular}{l|ccc|ccc|ccc}
% \hline
% \textbf{Fine-tuned on \textbackslash\ Tested on} & \multicolumn{3}{c|}{\textbf{DECORTE}} & \multicolumn{3}{c|}{\textbf{Karrierewege-syn-per-occupation}} & \multicolumn{3}{c}{\textbf{Karrierewege-syn-per-career-path}} \\
% \hline
% & \textbf{MRR} & \textbf{R@5} & \textbf{R@10} & \textbf{MRR} & \textbf{R@5} & \textbf{R@10} & \textbf{MRR} & \textbf{R@5} & \textbf{R@10} \\
% \hline
% \textbf{DECORTE} & \textbf{0.2394} & \textbf{0.3313} & \textbf{0.4008} & 0.7378 & 0.8123 & 0.8437 & 0.7816 & 0.8750 & 0.9099 \\
% \textbf{\textsc{DECORTE ESCO}} & 0.2153 & 0.2991 & 0.3718 & 0.7154 & 0.7903 & 0.8263 & 0.7361 & 0.8391 & 0.8815 \\
% \textbf{\textsc{Karrierewege}} & 0.1970 & 0.2636 & 0.3185 & 0.7393 & 0.8175 & 0.8546 & 0.7696 & 0.8583 & 0.8958 \\
% \textbf{\textsc{Karrierewege+}-per-occupation} & 0.2243 & 0.3124 & 0.3862 & \textbf{0.7827} & \textbf{0.8532} & \textbf{0.8898} & \underline{0.7825} & \underline{0.8754} & \underline{0.9152} \\
% \textbf{\textsc{Karrierewege+}-per-career-path} & \underline{0.2323} & \underline{0.3163} & \underline{0.3890} & \underline{0.7463} & \underline{0.8224} & \underline{0.8656} & \textbf{0.8238} & \textbf{0.9090} & \textbf{0.9392} \\
% \hline
% \end{tabular}
% }
% \caption{Cross-evaluation of models on different datasets}
% \label{tab:cross-evaluation}
% \end{table*} % with a mix: cecorte + len(decorte)* Karrierewege+-syn-per-career-path slightly better than original decorte
% In general, models trained on \textsc{Karrierewege} consistently show higher scores compared to those trained on DECORTE, even when they are trained on the same sized datasets (see results for 2k datasets in \ref{tab:results} This performance boost cannot be solely attributed to overlap between validation and test sets, as the overlap is quite low for \textsc{Karrierewege+}cp, (see Table \ref{tab:overlap} in the Appendix).
To better understand how training data size impacts performance, we experimented with multiple dataset sizes, allowing us to assess trends in model improvement across varying scales. Models trained on \textsc{Karrierewege} consistently achieve higher scores compared to those trained on DECORTE, even when the dataset sizes are identical (see Table \ref{tab:results}). This performance advantage cannot be solely attributed to validation and test set overlap, as the overlap remains negligible, or even minimal for \textsc{Karrierewege+}cp (see Table \ref{tab:overlap} in the Appendix).
This suggests that other patterns in the data, such as the more coherent career paths, might be contributing to the improved results. Across \textsc{Karrierewege}, \textsc{K+}oc, and \textsc{K+}cp, a clear performance improvement is observed with the use of larger training datasets. Notably, the performance increase is most pronounced when scaling from the 2k dataset to the 100k dataset, highlighting the significant impact of additional data. However, once the dataset size reaches a substantial volume, such as 100k, the performance gains taper off, as evidenced in the smaller improvement observed when scaling from 100k to 500k in \textsc{Karrierewege}.
\begin{table}
\centering
\resizebox{\columnwidth}{!}{
\begin{tabular}{l|ccc}
\hline
\textbf{Dataset} & \textbf{MRR} & \textbf{R@5} & \textbf{R@10} \\
\hline
\textbf{DECORTE 2k} & 0.2427 & 0.3418 & 0.4151 \\
\hdashline
\textbf{\textsc{K+}oc 2k} & 0.3846 & 0.4779 & 0.5423 \\
\textbf{\textsc{K+}oc 100k} & \textbf{0.4312} & \textbf{0.5340} & \textbf{0.6165} \\
\hdashline
\textbf{\textsc{K+}cp 2k} & 0.3702 & 0.4754 & 0.5568 \\
\textbf{\textsc{K+}cp 100k} & \textbf{0.4281} & \textbf{0.5280} & \textbf{0.6065} \\
\hline
\textbf{DECORTE ESCO 2k} & 0.2084 & 0.2813 & 0.3418 \\
\hdashline
\textbf{\textsc{Karrierewege} 2k} & 0.4232 & 0.5146 & 0.5636 \\
\textbf{\textsc{Karrierewege} 100k} & 0.4775 & 0.5671 & 0.6317 \\
\textbf{\textsc{Karrierewege} 500k} & \textbf{0.4867} & \textbf{0.5713} & \textbf{0.6347} \\
\hline
\end{tabular}
}
\caption{\label{tab:results}
Results for different free-text and standardized resume datasets with their approximate size.
}
\end{table}
Evaluating across datasets and synthesis methods shows that models trained on \textsc{Karrierewege+} datasets also performed well when tested on DECORTE, indicating that the paraphrased career paths generalize effectively across different datasets (see Table
% \ref{tab:cross-evaluation}
\ref{tab:combined_results}
). This strong performance suggests that the paraphrased data captures underlying patterns and relationships between job titles, making it adaptable across various contexts. Notably, models trained on \textsc{K+}cp datasets generalize better than models trained on \textsc{K+}oc, further supporting the idea that coherent career paths play a role in improving model performance.
%In cross-evaluation, the models trained on the free-text data significantly outperformed the models trained on the based taxonomy datasets (see Table \ref{tab:cross-evaluation}. The models trained on \textsc{Karrierewege+} datasets also performed well when tested on DECORTE, indicating that the paraphrased career paths generalize effectively across different datasets. This strong performance suggests that the paraphrased data captures underlying patterns and relationships between job titles, making it adaptable across various contexts.
% \begin{table*}[b!]
% \centering
% \resizebox{2\columnwidth}{!}{
% \begin{tabular}{l|ccc|ccc|ccc}
% \hline
% \textbf{Metric} & \multicolumn{3}{c|}{\textbf{MRR}} & \multicolumn{3}{c|}{\textbf{R@5}} & \multicolumn{3}{c}{\textbf{R@10}} \\
% \hline
% \textbf{Train/ Test} & \textbf{DECORTE} & \textbf{\textsc{K+}oc} & \textbf{\textsc{K+}cp} & \textbf{DECORTE} & \textbf{\textsc{K+}oc} & \textbf{\textsc{K+}cp} & \textbf{DECORTE} & \textbf{\textsc{K+}oc} & \textbf{\textsc{K+}cp} \\
% \hline
% \textbf{DECORTE} & \textbf{0.2427} & 0.1339 & 0.1588 & \textbf{0.3418} & 0.2005 & 0.2302 & \textbf{0.4151} & 0.2669 & 0.3076 \\
% \textbf{\textsc{K+}oc} & \underline{0.1303} & \textbf{0.4312} & \underline{0.3784} & 0.2164 & \textbf{0.5340} & 0.4899 & 0.3091 & \textbf{0.6165} & \underline{0.5784} \\
% \textbf{\textsc{K+}cp} & 0.1294 & \underline{0.3685} & \textbf{0.4281} & \underline{0.2186} & \underline{0.4693} & \textbf{0.5280} & \underline{0.3235} & \underline{0.5566} & \textbf{0.6065} \\
% \hline
% \end{tabular}
% }
% \caption{\label{tab:combined_results}
% Cross evaluation results for MRR, R@5, and R@10 across free-text datasets.
% }
% \end{table*}
\section{Conclusions and Further Research}
We introduced \textsc{Karrierewege} and \textsc{Karrierewege+}, large-scale, publicly available datasets for career path prediction. By linking the datasets to the ESCO taxonomy and synthesizing paraphrased job titles and descriptions, we addressed the challenge of predicting career trajectories from the free-text inputs typically found in resumes. Our results demonstrate that models trained on the \textsc{Karrierewege} datasets, particularly the \textsc{Karrierewege+}cp variant, perform well on free-text data, underscoring the importance of data diversity and richness for accurate career path prediction.
While these datasets provide a strong foundation for model training and evaluation, future work could focus on expanding their scope to include more regions, and languages, further enhancing their applicability to global career path prediction. Addressing challenges such as cross-industry career transitions could also improve model robustness and generalizability.
%While these datasets establish a robust foundation for model training and evaluation, future research could systematically investigate the expansion of their scope to include additional regions and languages, thereby increasing their relevance and applicability to global career path prediction tasks. Furthermore, addressing domain-specific challenges, such as modeling cross-industry career transitions and incorporating diverse labor market dynamics, could enhance the robustness and generalizability of prediction models.
%While these datasets provide a strong foundation for model training and evaluation, future work could focus on expanding their scope to include more regions, and languages, further enhancing their applicability to global career path prediction. Addressing challenges such as cross-industry career transitions could also improve model robustness and generalizability.
% \afterpage{\clearpage}
\FloatBarrier
% \newpage
\section*{Ethical Considerations}
The use of large-scale datasets like \textsc{Karrierewege} carries the risk of bias amplification if the dataset overrepresents certain industries, job levels, or demographics. Biases in the dataset could inadvertently lead to discriminatory predictions, particularly when applied to automated decision-making tools used by recruiters or employment agencies. Furthermore, synthesizing data to augment or enhance the dataset introduces additional risks, as the assumptions made by the underlying models may reflect or amplify existing biases. This could result in inaccurate or skewed descriptions of career trajectories, reinforcing stereotypes or marginalizing underrepresented groups.
To address these challenges, it is crucial to continuously monitor and mitigate biases that may emerge both in the original dataset and in any synthesized data. Strategies such as bias audits, fairness metrics, and diversification of training data sources should be implemented to ensure equitable model predictions.
%While these datasets provide a strong foundation for model training and evaluation, their focus could be expanded to include more regions and languages, further enhancing their applicability to global career path prediction.
\section*{Acknowledgements}
We thank the reviewers for their insightful feedback. We thank Jan-Peter Bergmann for computing infrastructure support and advice. ES and YC acknowledges financial support with funds provided by the German Federal Ministry for Economic Affairs and Climate Action due to an enactment of the German Bundestag under grant 46SKD127X (GENESIS). BP is supported by ERC Consolidator Grant DIALECT 101043235.
%Only 28 languages from the EU, no real low resource languages?
%This dataset encompasses resumes from individuals seeking employment across all industries. However, the dataset may be biased, as it possibly contains more resumes from industries with lower demand—where individuals are more inclined to register as unemployed—than from high-demand industries where unemployment registration is less common. Additionally, since all resumes are from individuals seeking employment in Germany, there is a cultural bias when applying this dataset to other regions.
\bibliography{custom}
\appendix
\section{Prompts for linkage}
\label{sec:prompts-linkage}
To address the limited context window of the GPT models, we used ISCO-08 codes as a filtering mechanism to narrow down potential occupation matches between Berufenet and ESCO. By applying the ISCO code, the number of candidate pairs was significantly reduced, allowing them to fit within the GPT models' context window. In some cases, however, better matches were found under different ISCO codes. Therefore, ISCO filtering was only applied when necessary to reduce the number of matches. Without this, each Berufenet occupation could potentially match up to 3,039 ESCO occupations.
\subsection{English Prompt}
\begin{lstlisting}
###CONTEXT###
You are a specialist in matching occupations with their ESCO labels. I have tried 3 different methods, and each method has a different prediction for the label. You will be presented with the occupation title, the 3 different predictions, and the set of candidate labels. The language is German.
###INSTRUCTION###
Choose the most appropriate label for an occupation title.
Return a JSON object with the job title and the selected label.
###DATA###
occupation_title: {occupation}
pred_1: {pred_1}
pred_2: {pred_2}
pred_3: {pred_3}
all_candidates: {candidates}
###OUTPUT###
\end{lstlisting}
\subsection{German Prompt}
\begin{lstlisting}
###CONTEXT###
Du bist ein Spezialist für das Matching von Berufen mit ihren ESCO-Labels. Ich habe 3 verschiedene Methoden ausprobiert, und jede Methode hat eine andere Vorhersage für das Label. Dir werden die Berufsbezeichnung, die 3 verschiedenen Vorhersagen und die Menge der in Frage kommenden Bezeichnungen vorgelegt. Die Sprache ist gemischt zwischen Deutsch.
###ANLEITUNG###
Wähle das am besten geeignete Label für eine Berufsbezeichnung.
Gib ein JSON-Objekt mit dem Berufstitel und dem gewählten Label zurück.
###DATA###
beruf_title: {occupation}
pred_1: {pred_1}
pred_2: {pred_2}
pred_3: {pred_3}
all_candidates: {candidates}
###OUTPUT###
\end{lstlisting}
\section{Prompts for title and description generation}
\label{sec:prompts-generation}
\subsection{Generation per career path}
\begin{lstlisting}
"""
Please create a paraphrased version of the following career path: {job_list}.
The length of the list should be the same as the original list. Please adhere to the format and do not add anything else.
###
Format:
'Career Path:
[paraphrased title 1, paraphrased title 2, ...]'
"""
\end{lstlisting}
\begin{lstlisting}
"""
Please create a description for the following jobs: {job_list}.
Please make it not longer than 4 sentences. Please adhere to the format and do not add anything else.
###
Format:
'Descriptions:
[Job title 1: Description of job 1,
Job title 2: Description of job 2,
...]'
"""
\end{lstlisting}
\subsection{Generation per occupation}
\begin{lstlisting}
"""
Paraphrase the following occupation title: {job}.
Please return a list of max 7 alternative job titles.
###
Example:
occupation title: physiotherapist
1. Physical Therapist
2. Physiotherapy Specialist
3. Rehabilitation Therapist
4. Movement Therapist
5. Injury Recovery Specialist
6. Musculoskeletal Therapist
7. Sports Medicine Therapist
"""
\end{lstlisting}
\begin{lstlisting}
"""
Please create a description for the following job: {job}.
Please make it not longer than 4 sentences. Please adhere to the format and do not add anything else.
###
Format:
'Description:
Job Description of the occupation...'
"""
\end{lstlisting}
\section{Prompts for LLM-evaluation}
\label{sec:prompts-evaluation}
\subsection{Prompt for all metrics at once}
\begin{lstlisting}
"""
# CONTEXT
A paraphrased career path should accurately reflect the skills and tasks of the original career path and it's job titles.
You will evaluate the paraphrased career path on the following four dimensions:
Correctness measures if the paraphrased titles are accurate representations of the original job titles.
A paraphrased career path with high correctness means that all titles are job titles that
exist in reality, but aren't just copies of
the original titles. A low-correctness paraphrased career path may contain titles that are not job titles or are the same title as the original title.
Semantic similarity assesses how well the paraphrased job titles captures the meaning of the original job titles in a career path.
A high semantic similarity score means that the paraphrased titles accurately represent the skills and tasks of the original titles.
Diversity measures how many unique job titles are present in the paraphrased career path.
A high diversity score means that the paraphrased career path contains a wide range of job titles and does not contain the same title multiple times.
Coherence evaluates how well the paraphrased job titles fit together within the paraphrased career path.
A highly coherent paraphrased career path will have job titles that make sense together or form a logical progression.
.
# INSTRUCTIONS
You will be presented with a original career path and its paraphrased version.
For each dimension, give the summary a score between 0 and 5. For correctness a score of 0 means
the paraphrased career path contains only job titles that aren't real job titles or just copies of the original title, while a score of 5 means it
provides only correct job titles.
For semantic similarity a score of 0 means that all paraphrased job titles are do not capture the meaning of the original titles very well, while a score of 5 means that all paraphrased job titles accurately represent the skills and tasks of the original titles.
For Diversity a score of 0 means that the paraphrased career path contains the same title multiple times, while a score of 5 means that the paraphrased career path contains a wide range of job titles.
For Coherence a score of 0 means that the paraphrased job titles do not make sense together or form a logical progression, while a score of 5 means that the paraphrased job titles fit together well in the career path.
Output the Likert scores for each dimension as a json (key: dimension, value: likert-score).
Do not add any explanation, answer only the Likert scores.
Use the initial marker ```json and the final marker ``` to mark the json content.\n\n
# EVALUATION MATERIALS
## Original career path
{original_career_path\}
## paraphrased career path
{paraphrased_career_path\}
"""
\end{lstlisting}
\subsection{Prompts for each metric}
\begin{lstlisting}
"""
# CONTEXT
Correctness measures if the paraphrased titles are accurate representations of the original job titles.
A paraphrased career path with high correctness means that all titles are job titles that
exist in reality, but aren't just copies of
the original titles. A low-correctness paraphrased career path may contain titles that are not job titles or are the same title as the original title.
# INSTRUCTIONS
You will be presented with a original career path and its paraphrased version.
Give the summary a score between 0 and 5.
Zero means the paraphrased career path contains only job titles that aren't real job titles or just copies of the original title,
while a score of 5 means it provides only correct job titles.
Just answer with the Likert Score, no text please.
# EVALUATION MATERIALS
## Original career path
{original_career_path}
## paraphrased career path
{paraphrased_career_path}
"""
\end{lstlisting}
\begin{lstlisting}
"""
# CONTEXT
Semantic similarity assesses how well the paraphrased job titles captures the meaning of the original job titles in a career path.
A high semantic similarity score means that the paraphrased titles accurately represent the skills and tasks of the original titles.
# INSTRUCTIONS
You will be presented with a original career path and its paraphrased version.
Give the summary a score between 0 and 5.
Zero means that all paraphrased job titles are do not capture the meaning of the original titles very well,
while a score of 5 means that all paraphrased job titles accurately represent the skills and tasks of the original titles.
Just answer with the Likert Score, no text please.
# EVALUATION MATERIALS
## Original career path
{original_career_path}
## paraphrased career path
{paraphrased_career_path}
"""
\end{lstlisting}
\begin{lstlisting}
"""
# CONTEXT
Diversity measures how many unique job titles are present in the paraphrased career path.
A high diversity score means that the paraphrased career path contains a wide range of job titles and does not contain the same title multiple times.
# INSTRUCTIONS
You will be presented with a paraphrased career path.
Give the summary a score between 0 and 5.
Zero means that the paraphrased career path contains the same title multiple times,
while a score of 5 means that the paraphrased career path contains a wide range of job titles.
Just answer with the Likert Score, no text please.
## paraphrased career path
{paraphrased_career_path}
"""
\end{lstlisting}
\begin{lstlisting}
"""
# CONTEXT
Coherence evaluates how well the paraphrased job titles fit together in the career path.
A highly coherent paraphrased career path will have job titles that make sense together or form a logical progression.
# INSTRUCTIONS
You will be presented with a paraphrased career path.
Give the summary a score between 0 and 5.
Zero means that the paraphrased job titles do not make sense together or form a logical progression,
while a score of 5 means that the paraphrased job titles fit together well in the career path.
Just answer with the Likert Score, no text please.
## paraphrased career path
{paraphrased_career_path}
"""
\end{lstlisting}
\section{Alignment Scores}
\label{sec:appendix_alignement_scores}
Table \ref{tab:llm-judge} shows the Likert scale score for the human labeling as well as the \code{gpt-4o-mini} results. Table \ref{tab:kappa} compares Kappa scores between \code{gpt-4o-mini} with one prompt and \code{gpt-4o-mini} with one prompt per metric. Table \ref{tab:rouge_bleu} presents BLEU and ROUGE-L scores for job titles and descriptions.
\begin{table}[h!]
\resizebox{\columnwidth}{!}{
\begin{tabular}{l|c|c}
\hline
\textbf{Metric} & \textbf{gpt-4o-mini one Prompt} & \textbf{gpt-4o-mini} \\ \hline
Correctness (CP) & 0.367089 & 0.058639 \\ \hline
Semantic Similarity (CP) & 0.116162 & -0.047056 \\ \hline
Diversity (CP) & 0.312162 & 0.264043 \\ \hline
Coherence (CP) & 0.082716 & -0.013099 \\ \hline
Correctness (Free) & 0.082840 & 0.100846 \\ \hline
Semantic Similarity (Free) & 0.022131 & -0.047251 \\ \hline
Diversity (Free) & 0.389831 & 0.333567 \\ \hline
Coherence (Free) & 0.022483 & -0.025326 \\ \hline
\end{tabular}
}
\caption{\label{tab:kappa}Comparison of Kappa scores between LLM with one prompt and LLM with one prompt per metric.}
\end{table}
\begin{table}[h!]
\resizebox{\columnwidth}{!}{
\begin{tabular}{l|c|c}
\hline
\textbf{Metric} & \textbf{\textsc{Karrierewege+}cp} & \textbf{\textsc{Karrierewege+}oc} \\ \hline
ROUGE-L Score (Job Titles) & 0.1618 & 0.1592 \\ \hline
ROUGE-L Score (Job Descriptions) & 0.0426 & 0.0233 \\ \hline
BLEU Score (Job Titles) & 0.0005 & 0.0002 \\ \hline
BLEU Score (Job Descriptions) & 0.0005 & 0.0003
\\ \hline
\end{tabular}
}
\caption{\label{tab:rouge_bleu}BLEU and ROUGE-L scores for job titles and descriptions.}
\end{table}
\begin{table*}
\resizebox{2\columnwidth}{!}{
\begin{tabular}{l|c|c|c|c}
\hline
\textbf{Metric} & \textbf{Manual Labeling (100)} & \textbf{gpt-4o-mini (100)} & \textbf{gpt-4o-mini one Prompt (100)} & \textbf{gpt-4o-mini one Prompt (all)}\\ \hline
Mean Correctness (CP) & 4.82 & 4.31 & 4.72 & 4.72 \\ \hline
Mean Correctness (OCC) & 4.72 & 4.26 & 4.74 & 4.70\\ \hline
Mean Semantic Similarity (CP) & 4.50 & 3.41 & 4.02 & 4.08 \\ \hline
Mean Semantic Similarity (OCC) & 4.20 & 2.91 & 3.91 & 3.85 \\ \hline
Mean Diversity (CP) & 4.08 & 3.38 & 4.01 & 3.85\\ \hline
Mean Diversity (OCC) & 4.43 & 3.88 & 4.33 & 4.22 \\ \hline
Mean Coherence (CP) & 4.01 & 1.58 & 4.04 & 4.12 \\ \hline
Mean Coherence (OCC) & 3.95 & 1.44 & 3.95 & 3.93 \\ \hline
\end{tabular}
}
\caption{\label{tab:llm-judge}Comparison of Mean Scores between Manual Labeling, the LLM-as-a-judge with one prompt per metric (gpt-4o-mini) and the LLM-as-a-judge with one one prompt for all metrics (gpt-4o-mini one Prompt).}
\end{table*}
\section{Experimental Setup}
\label{sec:exp_setup}
Following the recipe in \cite{decorte.2023}, we fine tune the model using Multiple Negatives Ranking Loss (MNRL), in-batch negatives and augmented career path data with all possible sub-paths of minimum length 2. This augmentation is applied after the data split. The fine-tuning process is conducted using a batch size of $16$, a learning rate of $2e-5$, for up to $1$ epoch for the large and $2$ epochs for the small datasets, with evaluation every $1$\% of steps based on validation loss. The best-performing model is saved based on these evaluations.
\section{Overlap Test and Validation Dataset}
\label{sec:overlap}
Table \ref{tab:overlap} shows the overlap between validation and test splits for various datasets.
\begin{table*}[h!]
\centering
\begin{tabular}{l|c|c|c|c}
\hline
\textbf{Dataset} & \textbf{Length Validation} & \textbf{Length Test} & \textbf{Overlap} & \textbf{Overlap \%} \\
\hline
DECORTE ESCO & 1,558 & 1,801 & 45 & 2.92\% \\
\hline
DECORTE & 1,558 & 1,801 & 0 & 0\% \\
\hline
\textsc{Karrierewege} & 667,404 & 658,012 & 56,101 & 8.53\% \\
\hline
\textsc{Karrierewege+}oc & 138,275 & 137,530 & 8,724 & 6.34\% \\
\hline
\textsc{Karrierewege+}cp & 138,275 & 137,530 & 139 & 0.10\% \\
\hline
\end{tabular}
\caption{\label{tab:overlap}
Overlap between validation and test splits across various datasets. The percentage is calculated as the number of overlapping entries divided by the total size of the test split.
}
\end{table*}
%\begin{figure*}[htbp]
% \centering
% \includegraphics[width=\textwidth]{data_processing_example.drawio.png}
% \caption{Sketch of a figure about the dataset creation process, needs improvement (I'm not very good with figures, will try to improve it later)}
% \label{fig:dataset_creation}
%\end{figure*}
\section{First level of the ESCO classification}
\label{sec:esco_first_level}
Table \ref{tab:esco_first_level} shows the first categorization level of the ESCO classification with their respective codes.
\begin{table}[h!]
\centering
\begin{tabular}{l|l}
\hline
\textbf{Code} & \textbf{First Level Occupation Category} \\
\hline
0 & Armed forces occupations \\
1 & Managers \\
2 & Professionals \\
3 & Technicians and associate \\
& professionals \\
4 & Clerical support workers \\
5 & Service and sales workers \\
6 & Skilled agricultural, forestry \\
& and fishery workers \\
7 & Craft and related trades workers \\
8 & Plant and machine operators \\
& and assemblers \\
9 & Elementary occupations \\
\hline
\end{tabular}
\caption{First level of the ESCO classification.}
\label{tab:esco_first_level}
\end{table}
\section{Absolute statistics of ESCO occupations distribution}
\label{sec:esco_occupations_concentration}
% The rank distribution plot in Figure \ref{fig:datasets_rank_plot} further highlights
% the distinctions on ESCO occupation coverage between \textsc{Karrierewege} and DECORTE datasets when considering full finer granilarity.
% On the distributions' head,
% \textsc{Karrierewege}'s most frequent occupations are in Sectors 9, 5, and 8, while DECORTE's are concentrated in Sectors 2 and 1.
% Moreover, \textsc{Karrierewege} shows a sharp concentration of its top occupations, while DECORTE has a more gradual decline in its top ranks, indicating a more balanced spread.
% Despite these differences in the top occupations, both datasets show similar coverage of less frequent occupations on the tail of their distributions.
% Nevertheless, in absolute terms, \textsc{Karrierewege} exhibits broader overall sector coverage, even within Sectors 1 and 2, as presented in Tables \ref{tab:karrierewege_level_1_dist} and \ref{tab:decorte_level_1_dist}.
% In absolute terms, \textsc{Karrierewege} exhibits broader overall sector coverage, even within Sectors 1 and 2,
As presented in Tables \ref{tab:karrierewege_level_1_dist} and \ref{tab:decorte_level_1_dist},
Sectors 1 and 2 have almost two order of magnitude higher absolute frequencies in \textsc{Karrierewege} when compared to DECORTE data.
By observing their absolute numbers, the massive difference between both datasets become clearer and highlights the potential of \textsc{Karrierewege}.
\begin{table}[h!]
\centering
\begin{tabular}{l|r}
\hline
\textbf{Code} & \vtop{\hbox{\strut \textbf{Absolute}}\hbox{\strut \textbf{Frequency}}} \\
\hline
9 & 941544 \\
5 & 379332 \\
7 & 247981 \\
2 & 230291 \\
3 & 213148 \\
8 & 194963 \\
4 & 139843 \\
1 & 106259 \\
6 & 23574 \\
0 & 3434 \\
\hline
\end{tabular}
\caption{Absolute frequency of first level ESCO occupations in \textsc{Karrierewege}.}
\label{tab:karrierewege_level_1_dist}
\end{table}
\begin{table}[h!]
\centering
\begin{tabular}{l|r}
\hline
\textbf{Code} & \vtop{\hbox{\strut \textbf{Absolute}}\hbox{\strut \textbf{Frequency}}} \\
\hline
2 & 2891 \\
1 & 2036 \\
3 & 1699 \\
5 & 807 \\
4 & 600 \\
7 & 210 \\
9 & 172 \\
8 & 66 \\
0 & 39 \\
6 & 12 \\
\hline
\end{tabular}
\caption{Absolute frequency of first level ESCO occupations in DECORTE.}
\label{tab:decorte_level_1_dist}
\end{table}
\end{document}
\begin{thebibliography}{15}
\providecommand{\natexlab}[1]{#1}
\bibitem[{Cerilla et~al.(2023)Cerilla, Santillan, Vinas, and Fuente}]{CerillaSVF23}
Micaela~Tayoto Cerilla, Aaron Santillan, Carl~John Vinas, and Michael B.~Dela Fuente. 2023.
\newblock Career path modeling and recommendations with linkedin career data and predicted salary estimations.
\newblock In \emph{The First Tiny Papers Track at {ICLR} 2023, Tiny Papers @ {ICLR} 2023, Kigali, Rwanda, May 5, 2023}. OpenReview.net.
\bibitem[{Chang et~al.(2019)Chang, Simon, Rajkumar, and Athey}]{Chang2019ABA}
Li-Te Chang, Lisa Simon, Karthik Rajkumar, and Susan Athey. 2019.
\newblock \href {https://api.semanticscholar.org/CorpusID:209521831} {A bayesian approach to predicting occupational transitions}.
\bibitem[{Decorte et~al.({2023})Decorte, Hautte, Deleu, Develder, and Demeester}]{decorte.2023}
Jens-Joris Decorte, Jeroen~Van Hautte, Johannes Deleu, Chris Develder, and Thomas Demeester. {2023}.
\newblock \href {{https://ceur-ws.org/Vol-3490/RecSysHR2023-paper_1.pdf}} {{Career path prediction using resume representation learning and skill-based matching}}.
\newblock In \emph{{RECSYS in HR 2023 : the 3rd Workshop on Recommender Systems for Human Resources (RecSys in HR 2023), Proceedings}}, volume {3490}, page~{9}. {CEUR}.
\bibitem[{Du et~al.(2024)Du, Kanodia, Brunborg, Vafa, and Athey}]{du2024laborllmlanguagebasedoccupationalrepresentations}
Tianyu Du, Ayush Kanodia, Herman Brunborg, Keyon Vafa, and Susan Athey. 2024.
\newblock \href {https://arxiv.org/abs/2406.17972} {Labor-llm: Language-based occupational representations with large language models}.
\newblock \emph{Preprint}, arXiv:2406.17972.
\bibitem[{Jayawardena and Yapa(2024)}]{Jayawardena_2024}
Lasal Jayawardena and Prasan Yapa. 2024.
\newblock \href {https://doi.org/10.5121/csit.2024.140418} {Parafusion: A large-scale llm-driven english paraphrase dataset infused with high-quality lexical and syntactic diversity}.
\newblock In \emph{Artificial Intelligence and Big Data}, AIBD, page 219–238. Academy \& Industry Research Collaboration Center.
\bibitem[{Li et~al.(2017)Li, Jing, Tong, Yang, He, and Chen}]{Li.2027}
Liangyue Li, How Jing, Hanghang Tong, Jaewon Yang, Qi~He, and Bee-Chung Chen. 2017.
\newblock \href {https://doi.org/10.1145/3041021.3054200} {Nemo: Next career move prediction with contextual embedding}.
\newblock In \emph{Proceedings of the 26th International Conference on World Wide Web Companion}, WWW '17 Companion, page 505–513, Republic and Canton of Geneva, CHE. International World Wide Web Conferences Steering Committee.
\bibitem[{Li et~al.(2023)Li, Kang, and Bie}]{li2023llm4jobsunsupervisedoccupationextraction}
Nan Li, Bo~Kang, and Tijl~De Bie. 2023.
\newblock \href {https://arxiv.org/abs/2309.09708} {Llm4jobs: Unsupervised occupation extraction and standardization leveraging large language models}.
\newblock \emph{Preprint}, arXiv:2309.09708.
\bibitem[{Lin(2004)}]{lin-2004-rouge}
Chin-Yew Lin. 2004.
\newblock \href {https://aclanthology.org/W04-1013} {{ROUGE}: A package for automatic evaluation of summaries}.
\newblock In \emph{Text Summarization Branches Out}, pages 74--81, Barcelona, Spain. Association for Computational Linguistics.
\bibitem[{Magron et~al.(2024)Magron, Dai, Zhang, Montariol, and Bosselut}]{magron-etal-2024-jobskape}
Antoine Magron, Anna Dai, Mike Zhang, Syrielle Montariol, and Antoine Bosselut. 2024.
\newblock \href {https://aclanthology.org/2024.nlp4hr-1.4} {{J}ob{S}kape: A framework for generating synthetic job postings to enhance skill matching}.
\newblock In \emph{Proceedings of the First Workshop on Natural Language Processing for Human Resources (NLP4HR 2024)}, pages 43--58, St. Julian{'}s, Malta. Association for Computational Linguistics.
\bibitem[{Papineni et~al.(2002)Papineni, Roukos, Ward, and Zhu}]{papineni-etal-2002-bleu}
Kishore Papineni, Salim Roukos, Todd Ward, and Wei-Jing Zhu. 2002.
\newblock \href {https://doi.org/10.3115/1073083.1073135} {{B}leu: a method for automatic evaluation of machine translation}.
\newblock In \emph{Proceedings of the 40th Annual Meeting of the Association for Computational Linguistics}, pages 311--318, Philadelphia, Pennsylvania, USA. Association for Computational Linguistics.
\bibitem[{Schellingerhout et~al.(2022)Schellingerhout, Medentsiy, and Marx}]{schellingerhout2022explainable}
Roan Schellingerhout, Volodymyr Medentsiy, and Maarten Marx. 2022.
\newblock Explainable career path predictions using neural models.
\newblock In \emph{HR@ RecSys}.
\bibitem[{Shreyas et~al.(2024)Shreyas, Praveen, Shreyas, and Kiran}]{Shreyas.2024}
R.~Shreyas, C.~Praveen, S.~Shreyas, and Y.C. Kiran. 2024.
\newblock \href {https://doi.org/10.48175/IJARSCT-15376} {A literature survey on ai driven career path prediction}.
\newblock \emph{International Journal of Advanced Research in Science, Communication and Technology}, pages 578--582.
\bibitem[{Thakur et~al.(2024)Thakur, Choudhary, Ramayapally, Vaidyanathan, and Hupkes}]{thakur2024judgingjudgesevaluatingalignment}
Aman~Singh Thakur, Kartik Choudhary, Venkat~Srinik Ramayapally, Sankaran Vaidyanathan, and Dieuwke Hupkes. 2024.
\newblock \href {https://arxiv.org/abs/2406.12624} {Judging the judges: Evaluating alignment and vulnerabilities in llms-as-judges}.
\newblock \emph{Preprint}, arXiv:2406.12624.
\bibitem[{Vafa et~al.(2024)Vafa, Palikot, Du, Kanodia, Athey, and Blei}]{vafa2024careerfoundationmodellabor}
Keyon Vafa, Emil Palikot, Tianyu Du, Ayush Kanodia, Susan Athey, and David~M. Blei. 2024.
\newblock \href {https://arxiv.org/abs/2202.08370} {Career: A foundation model for labor sequence data}.
\newblock \emph{Preprint}, arXiv:2202.08370.
\bibitem[{van~der Lee et~al.(2019)van~der Lee, Gatt, van Miltenburg, Wubben, and Krahmer}]{van-der-lee-etal-2019-best}
Chris van~der Lee, Albert Gatt, Emiel van Miltenburg, Sander Wubben, and Emiel Krahmer. 2019.
\newblock \href {https://doi.org/10.18653/v1/W19-8643} {Best practices for the human evaluation of automatically generated text}.
\newblock In \emph{Proceedings of the 12th International Conference on Natural Language Generation}, pages 355--368, Tokyo, Japan. Association for Computational Linguistics.
\end{thebibliography}