Почему RecyclerView странно ведет себя при выделении элементов?

<?xml version="1.0" encoding="utf-8"?>
<android.support.constraint.ConstraintLayout xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:tools="http://schemas.android.com/tools"
    android:layout_width="match_parent"
    android:layout_height="match_parent"
    tools:context="com.example.user.testactivity.MainActivity">

    <android.support.v7.widget.RecyclerView
        android:id="@+id/recyclerView"
        android:layout_width="match_parent"
        android:layout_height="match_parent">

    </android.support.v7.widget.RecyclerView>
</android.support.constraint.ConstraintLayout>

<LinearLayout xmlns:android="http://schemas.android.com/apk/res/android"
    xmlns:app="http://schemas.android.com/apk/res-auto"
    android:layout_width="match_parent"
    android:layout_height="wrap_content"
    android:orientation="vertical">

    <android.support.v7.widget.CardView
        android:id="@+id/card_view"
        android:layout_width="match_parent"
        android:layout_height="match_parent"
        app:contentPadding="10dp"
        app:cardUseCompatPadding="true">

        <LinearLayout
            android:layout_width="match_parent"
            android:layout_height="match_parent"
            android:paddingTop="16dp"
            android:paddingBottom="16dp"
            android:orientation="vertical">

            <TextView
                android:id="@+id/item_text"
                android:layout_width="match_parent"
                android:layout_height="wrap_content"
                android:textSize="20sp"
                android:textStyle="bold" />

        </LinearLayout>
    </android.support.v7.widget.CardView>
</LinearLayout>

package com.example.user.testactivity;

import android.os.Bundle;
import android.support.v7.app.AppCompatActivity;
import android.support.v7.widget.LinearLayoutManager;
import android.support.v7.widget.RecyclerView;

import java.util.ArrayList;
import java.util.List;

public class MainActivity extends AppCompatActivity {

    private RecyclerView recyclerView;
    private RecyclerView.LayoutManager layoutManager;
    private ItemAdapter itemAdapter;

    @Override
    protected void onCreate(Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.activity_main);
        recyclerView = findViewById(R.id.recyclerView);
        layoutManager = new LinearLayoutManager(this);
        recyclerView.setHasFixedSize(true);
        recyclerView.setLayoutManager(layoutManager);

        List<String> items = new ArrayList<>();
        for(int i = 0; i < 15; ++i){
            items.add("item "+i);
        }
        itemAdapter = new ItemAdapter(items);
        recyclerView.setAdapter(itemAdapter);
    }
}

package com.example.user.testactivity;

import android.graphics.Color;
import android.support.v7.widget.CardView;
import android.support.v7.widget.RecyclerView;
import android.util.Log;
import android.view.LayoutInflater;
import android.view.View;
import android.view.ViewGroup;
import android.widget.TextView;

import java.util.List;

public class ItemAdapter extends RecyclerView.Adapter<ItemViewHolder> {

    private List<String> mItems;

    public ItemAdapter(List<String> items) {
        mItems = items;
    }

    @Override
    public ItemViewHolder onCreateViewHolder(ViewGroup parent, int viewType) {
        View view = LayoutInflater.from(parent.getContext()).inflate(R.layout.row_item, parent, false);
        return new ItemViewHolder(view);
    }

    @Override
    public void onBindViewHolder(ItemViewHolder holder, int position) {
        final CardView cardView = holder.mCardView;
        holder.mTitle.setText(mItems.get(position));
        cardView.setTag(mItems.get(position));
    }


    @Override
    public int getItemCount() {
        return mItems.size();
    }
}

class ItemViewHolder extends RecyclerView.ViewHolder implements View.OnClickListener {
    TextView mTitle;
    CardView mCardView;

    public ItemViewHolder(View itemView) {
        super(itemView);
        mTitle = itemView.findViewById(R.id.item_text);
        mCardView = itemView.findViewById(R.id.card_view);
        mCardView.setOnClickListener(this);
    }

    @Override
    public void onClick(View v) {
        if(mCardView.isSelected()){
            Log.d("selectItem", "remove selection");
            mCardView.setCardBackgroundColor(Color.WHITE);
            mCardView.setSelected(false);
        }else{
            Log.d("selectItem", "set selection");
            mCardView.setCardBackgroundColor(Color.LTGRAY);
            mCardView.setSelected(true);
        }
    }
}