{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2026,4,13]],"date-time":"2026-04-13T12:58:57Z","timestamp":1776085137399,"version":"3.50.1"},"publisher-location":"New York, NY, USA","reference-count":39,"publisher":"ACM","license":[{"start":{"date-parts":[[2021,10,23]],"date-time":"2021-10-23T00:00:00Z","timestamp":1634947200000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/www.acm.org\/publications\/policies\/copyright_policy#Background"}],"content-domain":{"domain":["dl.acm.org"],"crossmark-restriction":true},"short-container-title":[],"published-print":{"date-parts":[[2021,10,23]]},"DOI":"10.1145\/3462204.3481729","type":"proceedings-article","created":{"date-parts":[[2021,10,22]],"date-time":"2021-10-22T22:38:59Z","timestamp":1634942339000},"page":"331-334","update-policy":"https:\/\/doi.org\/10.1145\/crossmark-policy","source":"Crossref","is-referenced-by-count":10,"title":["Investigating and Mitigating Biases in Crowdsourced Data"],"prefix":"10.1145","author":[{"given":"Danula","family":"Hettiachchi","sequence":"first","affiliation":[{"name":"RMIT University, Australia"}]},{"given":"Mark","family":"Sanderson","sequence":"additional","affiliation":[{"name":"RMIT University, Australia"}]},{"given":"Jorge","family":"Goncalves","sequence":"additional","affiliation":[{"name":"University of Melbourne, Australia"}]},{"given":"Simo","family":"Hosio","sequence":"additional","affiliation":[{"name":"University of Oulu, Finland"}]},{"given":"Gabriella","family":"Kazai","sequence":"additional","affiliation":[{"name":"Microsoft, United Kingdom"}]},{"given":"Matthew","family":"Lease","sequence":"additional","affiliation":[{"name":"University of Texas at Austin and Amazon, USA"}]},{"given":"Mike","family":"Schaekermann","sequence":"additional","affiliation":[{"name":"Amazon, Canada"}]},{"given":"Emine","family":"Yilmaz","sequence":"additional","affiliation":[{"name":"University College London and Amazon, United Kingdom"}]}],"member":"320","published-online":{"date-parts":[[2021,10,23]]},"reference":[{"key":"e_1_3_2_1_1_1","unstructured":"Hadis Anahideh Abolfazl Asudeh and Saravanan Thirumuruganathan. 2020. Fair active learning. Vol.\u00a01. ACM. arxiv:2001.01796  Hadis Anahideh Abolfazl Asudeh and Saravanan Thirumuruganathan. 2020. Fair active learning. Vol.\u00a01. ACM. arxiv:2001.01796"},{"key":"e_1_3_2_1_2_1","doi-asserted-by":"publisher","DOI":"10.1109\/ICDE.2019.00056"},{"key":"e_1_3_2_1_3_1","volume-title":"CEUR Workshop Proceedings 2276","author":"Balayn Agathe","year":"2018","unstructured":"Agathe Balayn , Panagiotis Mavridis , Alessandro Bozzon , Benjamin Timmermans , and Zolt\u00e1n Szl\u00e1vik . 2018 . Characterising and mitigating aggregation-bias in crowdsourced toxicity annotations . CEUR Workshop Proceedings 2276 (2018), 67\u201371. Agathe Balayn, Panagiotis Mavridis, Alessandro Bozzon, Benjamin Timmermans, and Zolt\u00e1n Szl\u00e1vik. 2018. Characterising and mitigating aggregation-bias in crowdsourced toxicity annotations. CEUR Workshop Proceedings 2276 (2018), 67\u201371."},{"key":"e_1_3_2_1_4_1","volume-title":"Conference on Human Factors in Computing Systems - Proceedings","author":"M.","year":"2019","unstructured":"Nat\u00e3\u00a0 M. Barbosa and Monchu Chen. 2019. Rehumanized crowdsourcing: A labeling framework addressing bias and ethics in machine learning . Conference on Human Factors in Computing Systems - Proceedings ( 2019 ), 1\u201312. https:\/\/doi.org\/10.1145\/3290605.3300773 10.1145\/3290605.3300773 Nat\u00e3\u00a0M. Barbosa and Monchu Chen. 2019. Rehumanized crowdsourcing: A labeling framework addressing bias and ethics in machine learning. Conference on Human Factors in Computing Systems - Proceedings (2019), 1\u201312. https:\/\/doi.org\/10.1145\/3290605.3300773"},{"key":"e_1_3_2_1_5_1","unstructured":"Flavio\u00a0P. Calmon Dennis Wei Karthikeyan\u00a0Natesan Ramamurthy and Kush\u00a0R. Varshney. 2017. Optimized data pre-processing for discrimination prevention. arXivNips(2017).  Flavio\u00a0P. Calmon Dennis Wei Karthikeyan\u00a0Natesan Ramamurthy and Kush\u00a0R. Varshney. 2017. Optimized data pre-processing for discrimination prevention. arXivNips(2017)."},{"key":"e_1_3_2_1_6_1","doi-asserted-by":"publisher","DOI":"10.1145\/3159652.3159654"},{"key":"e_1_3_2_1_7_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/D19-1107"},{"key":"e_1_3_2_1_8_1","volume-title":"Workshop on Fair and Responsible AI at ACM CHI","author":"Ghai Bhavya","year":"2020","unstructured":"Bhavya Ghai , Q.\u00a0 Vera Liao , Yunfeng Zhang , and Klaus Mueller . 2020 . Measuring Social Biases of Crowd Workers using Counterfactual Queries . In Workshop on Fair and Responsible AI at ACM CHI 2020. Bhavya Ghai, Q.\u00a0Vera Liao, Yunfeng Zhang, and Klaus Mueller. 2020. Measuring Social Biases of Crowd Workers using Counterfactual Queries. In Workshop on Fair and Responsible AI at ACM CHI 2020."},{"key":"e_1_3_2_1_9_1","doi-asserted-by":"publisher","DOI":"10.1145\/3306618.3314282"},{"key":"e_1_3_2_1_10_1","doi-asserted-by":"publisher","DOI":"10.1016\/j.comnet.2015.07.002"},{"key":"e_1_3_2_1_11_1","doi-asserted-by":"publisher","DOI":"10.1145\/3130916"},{"key":"e_1_3_2_1_12_1","doi-asserted-by":"publisher","DOI":"10.1145\/2818048.2819997"},{"key":"e_1_3_2_1_13_1","doi-asserted-by":"publisher","DOI":"10.1145\/3476073"},{"key":"e_1_3_2_1_14_1","doi-asserted-by":"publisher","DOI":"10.1145\/3415181"},{"key":"e_1_3_2_1_15_1","doi-asserted-by":"publisher","DOI":"10.1145\/2642918.2647362"},{"key":"e_1_3_2_1_16_1","doi-asserted-by":"publisher","DOI":"10.1109\/MC.2019.2902542"},{"key":"e_1_3_2_1_17_1","doi-asserted-by":"publisher","DOI":"10.1145\/3366423.3380063"},{"key":"e_1_3_2_1_18_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290605.3300637"},{"key":"e_1_3_2_1_19_1","doi-asserted-by":"publisher","DOI":"10.1609\/hcomp.v3i1.13238"},{"key":"#cr-split#-e_1_3_2_1_20_1.1","doi-asserted-by":"crossref","unstructured":"Faisal Kamiran and Toon Calders. 2012. Data preprocessing techniques for classification without discrimination. Vol.\u00a033. 1-33 pages. https:\/\/doi.org\/10.1007\/s10115-011-0463-8 10.1007\/s10115-011-0463-8","DOI":"10.1007\/s10115-011-0463-8"},{"key":"#cr-split#-e_1_3_2_1_20_1.2","doi-asserted-by":"crossref","unstructured":"Faisal Kamiran and Toon Calders. 2012. Data preprocessing techniques for classification without discrimination. Vol.\u00a033. 1-33 pages. https:\/\/doi.org\/10.1007\/s10115-011-0463-8","DOI":"10.1007\/s10115-011-0463-8"},{"key":"e_1_3_2_1_21_1","doi-asserted-by":"publisher","DOI":"10.1145\/3442381.3450080"},{"key":"e_1_3_2_1_22_1","doi-asserted-by":"publisher","DOI":"10.1145\/2441776.2441923"},{"key":"e_1_3_2_1_23_1","doi-asserted-by":"publisher","DOI":"10.1145\/3178876.3186133"},{"key":"e_1_3_2_1_24_1","volume-title":"Proceedings of the ACM SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation (CSE 2010","author":"Lease Matthew","year":"2010","unstructured":"Matthew Lease , Vitor Carvalho , and Emine Yilmaz ( Eds .). 2010 . Proceedings of the ACM SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation (CSE 2010 ). Online, Geneva, Switzerland. http:\/\/ir.ischool.utexas.edu\/cse 2010\/materials\/CSE2010-Proceedings.pdf Matthew Lease, Vitor Carvalho, and Emine Yilmaz (Eds.). 2010. Proceedings of the ACM SIGIR 2010 Workshop on Crowdsourcing for Search Evaluation (CSE 2010). Online, Geneva, Switzerland. http:\/\/ir.ischool.utexas.edu\/cse2010\/materials\/CSE2010-Proceedings.pdf"},{"key":"e_1_3_2_1_25_1","volume-title":"Proceedings of the text retrieval conference (TREC).","author":"Lease Matthew","year":"2011","unstructured":"Matthew Lease and Gabriella Kazai . 2011 . Overview of the trec 2011 crowdsourcing track . In Proceedings of the text retrieval conference (TREC). Matthew Lease and Gabriella Kazai. 2011. Overview of the trec 2011 crowdsourcing track. In Proceedings of the text retrieval conference (TREC)."},{"key":"e_1_3_2_1_26_1","doi-asserted-by":"publisher","DOI":"10.1007\/s10791-013-9222-7"},{"key":"e_1_3_2_1_27_1","unstructured":"Ninareh Mehrabi Fred Morstatter Nripsuta Saxena Kristina Lerman and Aram Galstyan. 2019. A survey on bias and fairness in machine learning. arxiv:1908.09635  Ninareh Mehrabi Fred Morstatter Nripsuta Saxena Kristina Lerman and Aram Galstyan. 2019. A survey on bias and fairness in machine learning. arxiv:1908.09635"},{"key":"e_1_3_2_1_28_1","volume-title":"Social Data: Biases, Methodological Pitfalls, and Ethical Boundaries. Frontiers in Big Data 2(2019). https:\/\/doi.org\/10.3389\/fdata.2019.00013","author":"Olteanu Alexandra","year":"2019","unstructured":"Alexandra Olteanu , Carlos Castillo , Fernando Diaz , and Emre K\u0131c\u0131man . 2019 . Social Data: Biases, Methodological Pitfalls, and Ethical Boundaries. Frontiers in Big Data 2(2019). https:\/\/doi.org\/10.3389\/fdata.2019.00013 10.3389\/fdata.2019.00013 Alexandra Olteanu, Carlos Castillo, Fernando Diaz, and Emre K\u0131c\u0131man. 2019. Social Data: Biases, Methodological Pitfalls, and Ethical Boundaries. Frontiers in Big Data 2(2019). https:\/\/doi.org\/10.3389\/fdata.2019.00013"},{"key":"e_1_3_2_1_29_1","doi-asserted-by":"publisher","DOI":"10.1145\/3290607.3299027"},{"key":"e_1_3_2_1_30_1","volume-title":"Accountability, Confidentiality, Transparency, and Safety in Information Retrieval. SIGIR Forum 53, 2","author":"Adam Roegiest","year":"2019","unstructured":"Adam Roegiest 2019. FACTS-IR : Fairness , Accountability, Confidentiality, Transparency, and Safety in Information Retrieval. SIGIR Forum 53, 2 ( 2019 ). Adam Roegiest 2019. FACTS-IR: Fairness, Accountability, Confidentiality, Transparency, and Safety in Information Retrieval. SIGIR Forum 53, 2 (2019)."},{"key":"e_1_3_2_1_31_1","doi-asserted-by":"publisher","DOI":"10.18653\/v1\/P19-1163"},{"key":"e_1_3_2_1_32_1","doi-asserted-by":"publisher","DOI":"10.1145\/3313831.3376290"},{"key":"e_1_3_2_1_33_1","doi-asserted-by":"publisher","DOI":"10.1145\/3274423"},{"key":"e_1_3_2_1_34_1","volume-title":"Proceedings of the 18th ACM Conference on Computer Supported Cooperative Work & Social Computing. 826\u2013838","author":"Sen Shilad","year":"2015","unstructured":"Shilad Sen , Margaret\u00a0 E Giesel , Rebecca Gold , Benjamin Hillmann , Matt Lesicko , Samuel Naden , Jesse Russell , Zixiao Wang , and Brent Hecht . 2015 . Turkers, Scholars,\u201cArafat\u201d and \u201cPeace\u201d Cultural Communities and Algorithmic Gold Standards . In Proceedings of the 18th ACM Conference on Computer Supported Cooperative Work & Social Computing. 826\u2013838 . Shilad Sen, Margaret\u00a0E Giesel, Rebecca Gold, Benjamin Hillmann, Matt Lesicko, Samuel Naden, Jesse Russell, Zixiao Wang, and Brent Hecht. 2015. Turkers, Scholars,\u201cArafat\u201d and \u201cPeace\u201d Cultural Communities and Algorithmic Gold Standards. In Proceedings of the 18th ACM Conference on Computer Supported Cooperative Work & Social Computing. 826\u2013838."},{"key":"e_1_3_2_1_35_1","doi-asserted-by":"publisher","DOI":"10.1145\/2766462.2767841"},{"key":"e_1_3_2_1_36_1","volume-title":"Proceedings of the 21st NIST Text Retrieval Conference (TREC).","author":"Smucker Mark","year":"2013","unstructured":"Mark Smucker , Gabriella Kazai , and Matthew Lease . 2013 . Overview of the TREC 2012 Crowdsourcing Track . In Proceedings of the 21st NIST Text Retrieval Conference (TREC). Mark Smucker, Gabriella Kazai, and Matthew Lease. 2013. Overview of the TREC 2012 Crowdsourcing Track. In Proceedings of the 21st NIST Text Retrieval Conference (TREC)."},{"key":"e_1_3_2_1_37_1","doi-asserted-by":"publisher","DOI":"10.1145\/3359130"},{"key":"e_1_3_2_1_38_1","doi-asserted-by":"publisher","DOI":"10.1145\/3411764.3445365"}],"event":{"name":"CSCW '21: Computer Supported Cooperative Work and Social Computing","location":"Virtual Event USA","acronym":"CSCW '21","sponsor":["SIGCHI ACM Special Interest Group on Computer-Human Interaction"]},"container-title":["Companion Publication of the 2021 Conference on Computer Supported Cooperative Work and Social Computing"],"original-title":[],"link":[{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3462204.3481729","content-type":"unspecified","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/dl.acm.org\/doi\/pdf\/10.1145\/3462204.3481729","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2025,6,17]],"date-time":"2025-06-17T20:17:01Z","timestamp":1750191421000},"score":1,"resource":{"primary":{"URL":"https:\/\/dl.acm.org\/doi\/10.1145\/3462204.3481729"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2021,10,23]]},"references-count":39,"alternative-id":["10.1145\/3462204.3481729","10.1145\/3462204"],"URL":"https:\/\/doi.org\/10.1145\/3462204.3481729","relation":{},"subject":[],"published":{"date-parts":[[2021,10,23]]},"assertion":[{"value":"2021-10-23","order":2,"name":"published","label":"Published","group":{"name":"publication_history","label":"Publication History"}}]}}