Due to the huge amount of parameters, fine-tuning of pretrained language models (PLMs) is prone to overfitting in the low-resource scenarios. In this work, we present a novel method that operates on the hidden representations of a PLM to reduce overfitting. During fine-tuning, our method inserts random autoencoders between the hidden layers of a PLM, which transform activations from the previous layers into a multi-view compressed representation before feeding it into the upper layers. The autoencoders are plugged out after fine-tuning, so our method does not add extra parameters or increase computation cost during inference. Our method demonstrates promising performance improvement across a wide range of sequence- and token-level low-resource NLP tasks. We will make our source code publicly available for research purposes.

@inproceedings{linlin-acl-23, abstract = {Due to the huge amount of parameters, fine-tuning of pretrained language models (PLMs) is prone to overfitting in the low-resource scenarios. In this work, we present a novel method that operates on the hidden representations of a PLM to reduce overfitting. During fine-tuning, our method inserts random autoencoders between the hidden layers of a PLM, which transform activations from the previous layers into a multi-view compressed representation before feeding it into the upper layers. The autoencoders are plugged out after fine-tuning, so our method does not add extra parameters or increase computation cost during inference. Our method demonstrates promising performance improvement across a wide range of sequence- and token-level low-resource NLP tasks. We will make our source code publicly available for research purposes.}, address = {Toronto, Canada}, author = {Linlin Liu and Xingxuan Li and Megh Thakkar and Xin Li and Shafiq Joty and Luo Si and Lidong Bing}, booktitle = {Proceedings of the 61st Annual Meeting of the Association for Computational Linguistics}, publisher = {ACL}, series = {ACL'23}, title = {Towards Robust Low-Resource Fine-Tuning with Multi-View Compressed Representations}, url = {https://arxiv.org/abs/2211.08794}, year = {2023} }